diff --git a/Cargo.toml b/Cargo.toml index 31e15b5..afd9d9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,5 +2,7 @@ resolver = '2' members = [ "source/postcard", - "source/postcard-derive", "source/postcard-schema", + "source/postcard-derive", + "source/postcard-dyn", + "source/postcard-schema", ] diff --git a/ci.sh b/ci.sh index 01a233a..7ca57a7 100755 --- a/ci.sh +++ b/ci.sh @@ -31,14 +31,19 @@ cargo_test() { cargo_test --features=alloc,experimental-derive,use-std,use-crc,derive -cargo_check --target=thumbv7em-none-eabi --no-default-features -cargo_check --target=thumbv7em-none-eabi --features=alloc,experimental-derive +# NOTE: we exclude postcard-dyn for these checks because it is std-only + +cargo_check --target=thumbv7em-none-eabi --no-default-features --exclude postcard-dyn +cargo_check --target=thumbv7em-none-eabi --features=alloc,experimental-derive --exclude postcard-dyn # CC https://github.com/jamesmunns/postcard/issues/167 - don't accidentally use atomics # on non-atomic systems -cargo_check --target=riscv32i-unknown-none-elf --features=alloc,experimental-derive +cargo_check --target=riscv32i-unknown-none-elf --features=alloc,experimental-derive --exclude postcard-dyn cargo fmt --all -- --check # Check docs.rs build -env RUSTDOCFLAGS='--cfg=docsrs --deny=warnings' cargo +nightly doc --all --no-deps --all-features +# +# TODO: We SHOULDN'T exclude postcard-dyn but it does weird things with feature unification and +# makes the embedded-io stuff break +env RUSTDOCFLAGS='--cfg=docsrs --deny=warnings' cargo +nightly doc --all --no-deps --all-features --exclude postcard-dyn diff --git a/source/postcard-dyn/.gitignore b/source/postcard-dyn/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/source/postcard-dyn/.gitignore @@ -0,0 +1 @@ +/target diff --git a/source/postcard-dyn/Cargo.toml b/source/postcard-dyn/Cargo.toml new file mode 100644 index 0000000..40d5394 --- /dev/null +++ b/source/postcard-dyn/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "postcard-dyn" +version = "0.1.0" +authors = ["James Munns "] +edition = "2021" +repository = "https://github.com/jamesmunns/postcard" +description = "Dynamic ser/de for postcard" +license = "MIT OR Apache-2.0" +categories = [ + "embedded", + "no-std", +] +keywords = [ + "serde", + "cobs", + "framing", +] +documentation = "https://docs.rs/postcard-dyn/" + + +[dependencies] +serde = { version = "1.0.202", features = ["derive"] } +serde_json = "1.0.117" + +[dependencies.postcard] +version = "1.0.10" +features = ["use-std"] +path = "../postcard" + +[dependencies.postcard-schema] +version = "0.1" +features = ["use-std", "derive"] +path = "../postcard-schema" diff --git a/source/postcard-dyn/src/de.rs b/source/postcard-dyn/src/de.rs new file mode 100644 index 0000000..3e06df3 --- /dev/null +++ b/source/postcard-dyn/src/de.rs @@ -0,0 +1,535 @@ +use std::str::from_utf8; + +use postcard_schema::schema::owned::{OwnedDataModelType, OwnedDataModelVariant, OwnedNamedType}; +use serde_json::{Map, Number, Value}; + +use crate::de::varint::de_zig_zag_i16; + +use self::varint::{ + de_zig_zag_i128, de_zig_zag_i32, de_zig_zag_i64, try_take_varint_u128, try_take_varint_u16, + try_take_varint_u32, try_take_varint_u64, try_take_varint_usize, +}; + +#[derive(Debug, PartialEq)] +pub enum Error { + UnexpectedEndOfData, + ShouldSupportButDont, + SchemaMismatch, +} + +trait GetExt { + type Out; + fn right(self) -> Result; +} + +impl GetExt for Option { + type Out = T; + + fn right(self) -> Result { + self.ok_or(Error::SchemaMismatch) + } +} + +pub fn from_slice_dyn(schema: &OwnedNamedType, data: &[u8]) -> Result { + let (val, _remain) = de_named_type(&schema.ty, data)?; + Ok(val) +} + +fn de_named_type<'a>(ty: &OwnedDataModelType, data: &'a [u8]) -> Result<(Value, &'a [u8]), Error> { + match ty { + OwnedDataModelType::Bool => { + let (one, rest) = data.take_one()?; + let val = match one { + 0 => Value::Bool(false), + 1 => Value::Bool(true), + _ => return Err(Error::SchemaMismatch), + }; + Ok((val, rest)) + } + OwnedDataModelType::I8 => { + let (one, rest) = data.take_one()?; + let val = Value::Number(Number::from(one as i8)); + Ok((val, rest)) + } + OwnedDataModelType::U8 => { + let (one, rest) = data.take_one()?; + let val = Value::Number(Number::from(one)); + Ok((val, rest)) + } + OwnedDataModelType::I16 => { + let (val, rest) = try_take_varint_u16(data)?; + let val = de_zig_zag_i16(val); + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::I32 => { + let (val, rest) = try_take_varint_u32(data)?; + let val = de_zig_zag_i32(val); + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::I64 => { + let (val, rest) = try_take_varint_u64(data)?; + let val = de_zig_zag_i64(val); + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::I128 => { + let (val, rest) = try_take_varint_u128(data)?; + let val = de_zig_zag_i128(val); + let val = i64::try_from(val).map_err(|_| Error::ShouldSupportButDont)?; + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::U16 => { + let (val, rest) = try_take_varint_u16(data)?; + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::U32 => { + let (val, rest) = try_take_varint_u32(data)?; + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::U64 => { + let (val, rest) = try_take_varint_u64(data)?; + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::U128 => { + let (val, rest) = try_take_varint_u128(data)?; + let val = u64::try_from(val).map_err(|_| Error::ShouldSupportButDont)?; + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::Usize => { + let (val, rest) = try_take_varint_usize(data)?; + let val = Value::Number(Number::from(val)); + Ok((val, rest)) + } + OwnedDataModelType::Isize => { + let (val, rest) = try_take_varint_usize(data)?; + + #[cfg(target_pointer_width = "16")] + let valu = de_zig_zag_i16(val as u16); + + #[cfg(target_pointer_width = "32")] + let valu = de_zig_zag_i32(val as u32); + + #[cfg(target_pointer_width = "64")] + let valu = de_zig_zag_i64(val as u64); + + let valu = Value::Number(Number::from(valu)); + Ok((valu, rest)) + } + OwnedDataModelType::F32 => { + let (val, rest) = data.take_n(4)?; + let mut buf = [0u8; 4]; + buf.copy_from_slice(val); + let f = f32::from_le_bytes(buf); + let val = Value::Number(Number::from_f64(f.into()).right()?); + Ok((val, rest)) + } + OwnedDataModelType::F64 => { + let (val, rest) = data.take_n(8)?; + let mut buf = [0u8; 8]; + buf.copy_from_slice(val); + let f = f64::from_le_bytes(buf); + let val = Value::Number(Number::from_f64(f).right()?); + Ok((val, rest)) + } + OwnedDataModelType::Char => todo!(), + OwnedDataModelType::String => { + let (val, rest) = try_take_varint_usize(data)?; + let (bytes, rest) = rest.take_n(val)?; + let s = from_utf8(bytes).map_err(|_| Error::SchemaMismatch)?; + let val = Value::String(s.to_string()); + Ok((val, rest)) + } + OwnedDataModelType::ByteArray => { + let (val, rest) = try_take_varint_usize(data)?; + let (bytes, rest) = rest.take_n(val)?; + let vvec = bytes + .iter() + .map(|b| Value::Number(Number::from(*b))) + .collect::>(); + let val = Value::Array(vvec); + Ok((val, rest)) + } + OwnedDataModelType::Option(nt) => { + let (val, rest) = data.take_one()?; + match val { + 0 => return Ok((Value::Null, rest)), + 1 => {} + _ => return Err(Error::SchemaMismatch), + } + de_named_type(&nt.ty, rest) + } + OwnedDataModelType::Unit | OwnedDataModelType::UnitStruct => { + // TODO This is PROBABLY wrong, as Some(()) will be coalesced into the same + // value as None. Fix this when we have our own Value + Ok((Value::Null, data)) + } + OwnedDataModelType::NewtypeStruct(nt) => de_named_type(&nt.ty, data), + OwnedDataModelType::Seq(nt) => { + let (val, mut rest) = try_take_varint_usize(data)?; + let mut vec = vec![]; + for _ in 0..val { + let (v, irest) = de_named_type(&nt.ty, rest)?; + rest = irest; + vec.push(v); + } + Ok((Value::Array(vec), rest)) + } + OwnedDataModelType::Tuple(nts) | OwnedDataModelType::TupleStruct(nts) => { + match nts.as_slice() { + [] => { + // TODO: Not sure this is right... + Ok((Value::Null, data)) + } + [nt] => { + // Single item, NOT an array + de_named_type(&nt.ty, data) + } + multi => { + let mut vec = vec![]; + let mut rest = data; + for nt in multi.iter() { + let (val, irest) = de_named_type(&nt.ty, rest)?; + rest = irest; + vec.push(val); + } + Ok((Value::Array(vec), rest)) + } + } + } + OwnedDataModelType::Map { key, val } => { + // TODO: impling blind because we can't test this, oops + // + // TODO: There's also a mismatch here because serde_json::Value requires + // keys to be strings, when postcard doesn't. + if key.ty != OwnedDataModelType::String { + return Err(Error::ShouldSupportButDont); + } + + let (map_len, mut rest) = try_take_varint_usize(data)?; + let mut map = Map::new(); + + for _ in 0..map_len { + let (str_len, irest) = try_take_varint_usize(rest)?; + let (bytes, irest) = irest.take_n(str_len)?; + let s = from_utf8(bytes).map_err(|_| Error::SchemaMismatch)?; + + let (v, irest) = de_named_type(&val.ty, irest)?; + rest = irest; + + map.insert(s.to_string(), v); + } + + Ok((Value::Object(map), rest)) + } + OwnedDataModelType::Struct(nvs) => { + let mut map = Map::new(); + let mut rest = data; + for nv in nvs.iter() { + let (val, irest) = de_named_type(&nv.ty.ty, rest)?; + rest = irest; + map.insert(nv.name.to_string(), val); + } + Ok((Value::Object(map), rest)) + } + OwnedDataModelType::Enum(nvars) => { + let (variant, rest) = try_take_varint_usize(data)?; + let schema = nvars.get(variant).right()?; + match &schema.ty { + OwnedDataModelVariant::UnitVariant => { + // Units become strings + Ok((Value::String(schema.name.to_string()), rest)) + } + OwnedDataModelVariant::NewtypeVariant(owned_named_type) => { + // everything else becomes an object with one field + let (val, irest) = de_named_type(&owned_named_type.ty, rest)?; + let mut map = Map::new(); + map.insert(schema.name.to_owned().to_string(), val); + Ok((Value::Object(map), irest)) + } + OwnedDataModelVariant::TupleVariant(vec) => { + // everything else becomes an object with one field + let (val, irest) = + de_named_type(&OwnedDataModelType::Tuple(vec.clone()), rest)?; + let mut map = Map::new(); + map.insert(schema.name.to_owned().to_string(), val); + Ok((Value::Object(map), irest)) + } + OwnedDataModelVariant::StructVariant(vec) => { + // everything else becomes an object with one field + let (val, irest) = + de_named_type(&OwnedDataModelType::Struct(vec.clone()), rest)?; + let mut map = Map::new(); + map.insert(schema.name.to_owned().to_string(), val); + Ok((Value::Object(map), irest)) + } + } + } + OwnedDataModelType::Schema => todo!(), + } +} + +mod varint { + // copy and paste from postcard + + use crate::ser::varint::varint_max; + + use super::{Error, TakeExt}; + + /// Returns the maximum value stored in the last encoded byte. + pub const fn max_of_last_byte() -> u8 { + let max_bits = core::mem::size_of::() * 8; + let extra_bits = max_bits % 7; + (1 << extra_bits) - 1 + } + + pub fn de_zig_zag_i16(n: u16) -> i16 { + ((n >> 1) as i16) ^ (-((n & 0b1) as i16)) + } + + pub fn de_zig_zag_i32(n: u32) -> i32 { + ((n >> 1) as i32) ^ (-((n & 0b1) as i32)) + } + + pub fn de_zig_zag_i64(n: u64) -> i64 { + ((n >> 1) as i64) ^ (-((n & 0b1) as i64)) + } + + pub fn de_zig_zag_i128(n: u128) -> i128 { + ((n >> 1) as i128) ^ (-((n & 0b1) as i128)) + } + + #[cfg(target_pointer_width = "16")] + #[inline(always)] + pub fn try_take_varint_usize(data: &[u8]) -> Result<(usize, &[u8]), Error> { + try_take_varint_u16(data).map(|(u, rest)| (u as usize, rest)) + } + + #[cfg(target_pointer_width = "32")] + #[inline(always)] + pub fn try_take_varint_usize(data: &[u8]) -> Result<(usize, &[u8]), Error> { + try_take_varint_u32(data).map(|(u, rest)| (u as usize, rest)) + } + + #[cfg(target_pointer_width = "64")] + #[inline(always)] + pub fn try_take_varint_usize(data: &[u8]) -> Result<(usize, &[u8]), Error> { + try_take_varint_u64(data).map(|(u, rest)| (u as usize, rest)) + } + + #[inline] + pub fn try_take_varint_u16(data: &[u8]) -> Result<(u16, &[u8]), Error> { + let mut rest = data; + let mut out = 0; + for i in 0..varint_max::() { + let (val, later) = rest.take_one()?; + rest = later; + let carry = (val & 0x7F) as u16; + out |= carry << (7 * i); + + if (val & 0x80) == 0 { + if i == varint_max::() - 1 && val > max_of_last_byte::() { + return Err(Error::SchemaMismatch); + } else { + return Ok((out, rest)); + } + } + } + Err(Error::SchemaMismatch) + } + + #[inline] + pub fn try_take_varint_u32(data: &[u8]) -> Result<(u32, &[u8]), Error> { + let mut rest = data; + let mut out = 0; + for i in 0..varint_max::() { + let (val, later) = rest.take_one()?; + rest = later; + let carry = (val & 0x7F) as u32; + out |= carry << (7 * i); + + if (val & 0x80) == 0 { + if i == varint_max::() - 1 && val > max_of_last_byte::() { + return Err(Error::SchemaMismatch); + } else { + return Ok((out, rest)); + } + } + } + Err(Error::SchemaMismatch) + } + + #[inline] + pub fn try_take_varint_u64(data: &[u8]) -> Result<(u64, &[u8]), Error> { + let mut rest = data; + let mut out = 0; + for i in 0..varint_max::() { + let (val, later) = rest.take_one()?; + rest = later; + let carry = (val & 0x7F) as u64; + out |= carry << (7 * i); + + if (val & 0x80) == 0 { + if i == varint_max::() - 1 && val > max_of_last_byte::() { + return Err(Error::SchemaMismatch); + } else { + return Ok((out, rest)); + } + } + } + Err(Error::SchemaMismatch) + } + + #[inline] + pub fn try_take_varint_u128(data: &[u8]) -> Result<(u128, &[u8]), Error> { + let mut rest = data; + let mut out = 0; + for i in 0..varint_max::() { + let (val, later) = rest.take_one()?; + rest = later; + let carry = (val & 0x7F) as u128; + out |= carry << (7 * i); + + if (val & 0x80) == 0 { + if i == varint_max::() - 1 && val > max_of_last_byte::() { + return Err(Error::SchemaMismatch); + } else { + return Ok((out, rest)); + } + } + } + Err(Error::SchemaMismatch) + } +} + +trait TakeExt { + fn take_one(&self) -> Result<(u8, &[u8]), Error>; + fn take_n(&self, n: usize) -> Result<(&[u8], &[u8]), Error>; +} + +impl TakeExt for [u8] { + fn take_one(&self) -> Result<(u8, &[u8]), Error> { + if let Some((first, rest)) = self.split_first() { + Ok((*first, rest)) + } else { + Err(Error::UnexpectedEndOfData) + } + } + + fn take_n(&self, n: usize) -> Result<(&[u8], &[u8]), Error> { + if self.len() < n { + return Err(Error::UnexpectedEndOfData); + } + Ok(self.split_at(n)) + } +} + +#[cfg(test)] +mod test { + use postcard_schema::Schema; + use serde::{Deserialize, Serialize}; + use serde_json::json; + + use crate::{from_slice_dyn, to_stdvec_dyn}; + + #[derive(Serialize, Deserialize, Schema)] + struct UnitStruct; + + #[derive(Serialize, Deserialize, Schema)] + struct TupStruct1(u8); + + #[derive(Serialize, Deserialize, Schema)] + struct TupStruct2(u8, u8); + + #[derive(Serialize, Deserialize, Schema)] + struct Struct1 { + pub x: bool, + pub y: u16, + pub z: f64, + } + + #[derive(Serialize, Deserialize, Schema)] + enum Enum1 { + Alpha, + Beta(u8), + Gamma(Vec), + Delta(Struct1), + Epsilon(u8, bool), + // TODO: struct variants are broken in the Schema derive in + // stable postcard, tho it is fixed on the main branch. + // Zeta { a: u8, b: bool }, + } + + #[test] + fn smoke() { + let bye = serde_json::to_value(Enum1::Alpha).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![0], t); + let de = from_slice_dyn(&Enum1::SCHEMA.into(), &t).unwrap(); + assert_eq!( + de, + json! { + "Alpha" + } + ); + + let bye = serde_json::to_value(Enum1::Beta(4)).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![1, 4], t); + let de = from_slice_dyn(&Enum1::SCHEMA.into(), &t).unwrap(); + assert_eq!( + de, + json! { + {"Beta": 4} + } + ); + + let bye = serde_json::to_value(Enum1::Gamma(vec![1, 2, 3])).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![2, 3, 1, 2, 3], t); + let de = from_slice_dyn(&Enum1::SCHEMA.into(), &t).unwrap(); + assert_eq!( + de, + json! { + {"Gamma": [1, 2, 3]} + } + ); + + let bye = serde_json::to_value(Enum1::Delta(Struct1 { + x: false, + y: 1000, + z: 4.0, + })) + .unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![3, 0, 232, 7, 0, 0, 0, 0, 0, 0, 16, 64], t); + let de = from_slice_dyn(&Enum1::SCHEMA.into(), &t).unwrap(); + assert_eq!( + de, + json! { + {"Delta": { + "x": false, + "y": 1000, + "z": 4.0 + }} + } + ); + + let bye = serde_json::to_value(Enum1::Epsilon(8, false)).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![4, 8, 0], t); + let de = from_slice_dyn(&Enum1::SCHEMA.into(), &t).unwrap(); + assert_eq!( + de, + json! { + {"Epsilon": [8, false]} + } + ); + } +} diff --git a/source/postcard-dyn/src/lib.rs b/source/postcard-dyn/src/lib.rs new file mode 100644 index 0000000..2316cf9 --- /dev/null +++ b/source/postcard-dyn/src/lib.rs @@ -0,0 +1,6 @@ +mod de; +mod ser; + +pub use de::from_slice_dyn; +pub use ser::to_stdvec_dyn; +pub use serde_json::Value; diff --git a/source/postcard-dyn/src/ser.rs b/source/postcard-dyn/src/ser.rs new file mode 100644 index 0000000..dd75fcd --- /dev/null +++ b/source/postcard-dyn/src/ser.rs @@ -0,0 +1,704 @@ +use std::num::TryFromIntError; + +use postcard_schema::schema::owned::{OwnedDataModelType, OwnedDataModelVariant, OwnedNamedType}; +use serde_json::Value; +use varint::{ + varint_max, varint_u128, varint_u16, varint_u32, varint_u64, varint_usize, zig_zag_i128, + zig_zag_i16, zig_zag_i32, zig_zag_i64, +}; + +#[derive(Debug, PartialEq)] +pub enum Error { + SchemaMismatch, + /// Limitations of using serde_json::Value for now + ShouldSupportButDont, + Unsupported, +} + +pub fn to_stdvec_dyn(schema: &OwnedNamedType, value: &Value) -> Result, Error> { + let mut out = vec![]; + + ser_named_type(&schema.ty, value, &mut out)?; + + Ok(out) +} + +trait GetExt { + type Out; + fn right(self) -> Result; +} + +impl GetExt for Option { + type Out = T; + + fn right(self) -> Result { + self.ok_or(Error::SchemaMismatch) + } +} + +impl From for Error { + fn from(_: TryFromIntError) -> Self { + Self::SchemaMismatch + } +} + +fn ser_named_type(ty: &OwnedDataModelType, value: &Value, out: &mut Vec) -> Result<(), Error> { + match ty { + OwnedDataModelType::Bool => { + let val = value.as_bool().right()?; + out.push(if val { 0x01 } else { 0x00 }); + } + OwnedDataModelType::I8 => { + let val = value.as_i64().right()?; + let val = i8::try_from(val)?; + out.push(val as u8); + } + OwnedDataModelType::U8 => { + let val = value.as_u64().right()?; + let val = u8::try_from(val)?; + out.push(val); + } + OwnedDataModelType::I16 => { + let val = value.as_i64().right()?; + let val = i16::try_from(val)?; + let val = zig_zag_i16(val); + let mut buf = [0u8; varint_max::()]; + let used = varint_u16(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::I32 => { + let val = value.as_i64().right()?; + let val = i32::try_from(val)?; + let val = zig_zag_i32(val); + let mut buf = [0u8; varint_max::()]; + let used = varint_u32(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::I64 => { + let val = value.as_i64().right()?; + let val = zig_zag_i64(val); + let mut buf = [0u8; varint_max::()]; + let used = varint_u64(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::I128 => { + let val = value.as_i64().right()?; + let val = i128::from(val); + let val = zig_zag_i128(val); + let mut buf = [0u8; varint_max::()]; + let used = varint_u128(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::U16 => { + let val = value.as_u64().right()?; + let val = u16::try_from(val)?; + let mut buf = [0u8; varint_max::()]; + let used = varint_u16(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::U32 => { + let val = value.as_u64().right()?; + let val = u32::try_from(val)?; + let mut buf = [0u8; varint_max::()]; + let used = varint_u32(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::U64 => { + let val = value.as_u64().right()?; + let mut buf = [0u8; varint_max::()]; + let used = varint_u64(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::U128 => { + let val = value.as_u64().right()?; + let val = u128::from(val); + let mut buf = [0u8; varint_max::()]; + let used = varint_u128(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::Usize => { + let val = value.as_u64().right()?; + let val = usize::try_from(val)?; + let mut buf = [0u8; varint_max::()]; + let used = varint_usize(val, &mut buf); + out.extend_from_slice(used); + } + OwnedDataModelType::Isize => { + let val = value.as_i64().right()?; + let mut buf; + let used; + + // hax + #[cfg(target_pointer_width = "16")] + { + let val = i16::try_from(val)?; + let val = zig_zag_i16(val); + buf = [0u8; varint_max::()]; + used = varint_u16(val, &mut buf); + } + #[cfg(target_pointer_width = "32")] + { + let val = i32::try_from(val)?; + let val = zig_zag_i32(val); + buf = [0u8; varint_max::()]; + used = varint_u32(val, &mut buf); + } + #[cfg(target_pointer_width = "64")] + { + let val = zig_zag_i64(val); + buf = [0u8; varint_max::()]; + used = varint_u64(val, &mut buf); + } + out.extend_from_slice(used); + } + OwnedDataModelType::F32 => { + let val = value.as_f64().right()?; + let val = val as f32; // todo + let val = val.to_le_bytes(); + out.extend_from_slice(&val); + } + OwnedDataModelType::F64 => { + let val = value.as_f64().right()?; + let val = val.to_le_bytes(); + out.extend_from_slice(&val); + } + OwnedDataModelType::String | OwnedDataModelType::Char => { + let val = value.as_str().right()?; + + // First add len + let len = val.len(); + let mut buf = [0u8; varint_max::()]; + let used = varint_usize(len, &mut buf); + out.extend_from_slice(used); + + // Then add payload + out.extend_from_slice(val.as_bytes()); + } + OwnedDataModelType::ByteArray => { + let val = value.as_array().right()?; + + // First add len + let len = val.len(); + let mut buf = [0u8; varint_max::()]; + let used = varint_usize(len, &mut buf); + out.extend_from_slice(used); + + // Then add values + for b in val { + let val = b.as_u64().right()?; + let val = u8::try_from(val)?; + out.push(val); + } + } + OwnedDataModelType::Option(nt) => { + if value.is_null() { + out.push(0x00); + } else { + out.push(0x01); + ser_named_type(&nt.ty, value, out)?; + } + } + OwnedDataModelType::Unit => {} + OwnedDataModelType::UnitStruct => {} + OwnedDataModelType::NewtypeStruct(nt) => { + ser_named_type(&nt.ty, value, out)?; + } + OwnedDataModelType::Seq(nt) => { + let val = value.as_array().right()?; + + // First add len + let len = val.len(); + let mut buf = [0u8; varint_max::()]; + let used = varint_usize(len, &mut buf); + out.extend_from_slice(used); + + // Then add values + for b in val { + ser_named_type(&nt.ty, b, out)?; + } + } + OwnedDataModelType::Tuple(nts) | OwnedDataModelType::TupleStruct(nts) => { + // Tuples with arity of 1 are not arrays, but instead just a single object + if nts.len() == 1 { + return ser_named_type(&nts[0].ty, value, out); + } + + let val = value.as_array().right()?; + + if val.len() != nts.len() { + return Err(Error::SchemaMismatch); + } + + for (nt, val) in nts.iter().zip(val.iter()) { + ser_named_type(&nt.ty, val, out)?; + } + } + OwnedDataModelType::Map { key, val } => { + // TODO: impling blind because we can't test this, oops + // + // TODO: There's also a mismatch here because serde_json::Value requires + // keys to be strings, when postcard doesn't. + if key.ty != OwnedDataModelType::String { + return Err(Error::ShouldSupportButDont); + } + + let obj = value.as_object().right()?; + + // First add len + let len = obj.len(); + let mut buf = [0u8; varint_max::()]; + let used = varint_usize(len, &mut buf); + out.extend_from_slice(used); + + // Then for each pair, serialize key then val + for (k, v) in obj.iter() { + // KEY + // + // First add len + let len = k.len(); + let used = varint_usize(len, &mut buf); + out.extend_from_slice(used); + + // Then add payload + out.extend_from_slice(k.as_bytes()); + + // VALUE + ser_named_type(&val.ty, v, out)?; + } + } + OwnedDataModelType::Struct(nvs) => { + let val = value.as_object().right()?; + + if val.len() != nvs.len() { + return Err(Error::SchemaMismatch); + } + + for field in nvs.iter() { + let v = val.get(&field.name).right()?; + ser_named_type(&field.ty.ty, v, out)?; + } + } + OwnedDataModelType::Enum(nvars) => { + // This is a bit serde_json::Value specific, if we make our own value + // type we might be able to handle this "better" + + // Is this a valueless variant? + if let Some(s) = value.as_str() { + // Is there a unit variant that matches this name? + let (idx, evar) = nvars + .iter() + .enumerate() + .find(|(_i, v)| v.name == s) + .right()?; + if evar.ty != OwnedDataModelVariant::UnitVariant { + return Err(Error::SchemaMismatch); + } + + // cool, we found it, serialize as a varint usize + let mut buf = [0u8; varint_max::()]; + let used = varint_usize(idx, &mut buf); + out.extend_from_slice(used); + } else if let Some(o) = value.as_object() { + // This should be an object with exactly one key + if o.len() != 1 { + return Err(Error::SchemaMismatch); + } + let (k, v) = o.iter().next().right()?; + let (idx, evar) = nvars + .iter() + .enumerate() + .find(|(_i, v)| &v.name == k) + .right()?; + + // cool, we found it, serialize as a varint usize + let mut buf = [0u8; varint_max::()]; + let used = varint_usize(idx, &mut buf); + out.extend_from_slice(used); + + // then serialize the value + match &evar.ty { + OwnedDataModelVariant::UnitVariant => { + // Nothing to do + } + OwnedDataModelVariant::NewtypeVariant(owned_named_type) => { + ser_named_type(&owned_named_type.ty, v, out)?; + } + OwnedDataModelVariant::TupleVariant(nts) => { + // Tuples with arity of 1 are not arrays, but instead just a single object + if nts.len() == 1 { + return ser_named_type(&nts[0].ty, v, out); + } + + let val = v.as_array().right()?; + + if val.len() != nts.len() { + return Err(Error::SchemaMismatch); + } + + for (nt, val) in nts.iter().zip(val.iter()) { + ser_named_type(&nt.ty, val, out)?; + } + } + OwnedDataModelVariant::StructVariant(nvs) => { + let val = v.as_object().right()?; + + if val.len() != nvs.len() { + return Err(Error::SchemaMismatch); + } + + for field in nvs.iter() { + let v = val.get(&field.name).right()?; + ser_named_type(&field.ty.ty, v, out)?; + } + } + } + } else { + return Err(Error::SchemaMismatch); + } + } + OwnedDataModelType::Schema => todo!(), + } + Ok(()) +} + +pub(crate) mod varint { + // copy and paste from postcard + + /// Returns the maximum number of bytes required to encode T. + pub const fn varint_max() -> usize { + const BITS_PER_BYTE: usize = 8; + const BITS_PER_VARINT_BYTE: usize = 7; + + // How many data bits do we need for this type? + let bits = core::mem::size_of::() * BITS_PER_BYTE; + + // We add (BITS_PER_VARINT_BYTE - 1), to ensure any integer divisions + // with a remainder will always add exactly one full byte, but + // an evenly divided number of bits will be the same + let roundup_bits = bits + (BITS_PER_VARINT_BYTE - 1); + + // Apply division, using normal "round down" integer division + roundup_bits / BITS_PER_VARINT_BYTE + } + + #[inline] + pub fn varint_usize(n: usize, out: &mut [u8; varint_max::()]) -> &mut [u8] { + let mut value = n; + for i in 0..varint_max::() { + out[i] = value.to_le_bytes()[0]; + if value < 128 { + return &mut out[..=i]; + } + + out[i] |= 0x80; + value >>= 7; + } + debug_assert_eq!(value, 0); + &mut out[..] + } + + #[inline] + pub fn varint_u16(n: u16, out: &mut [u8; varint_max::()]) -> &mut [u8] { + let mut value = n; + for i in 0..varint_max::() { + out[i] = value.to_le_bytes()[0]; + if value < 128 { + return &mut out[..=i]; + } + + out[i] |= 0x80; + value >>= 7; + } + debug_assert_eq!(value, 0); + &mut out[..] + } + + #[inline] + pub fn varint_u32(n: u32, out: &mut [u8; varint_max::()]) -> &mut [u8] { + let mut value = n; + for i in 0..varint_max::() { + out[i] = value.to_le_bytes()[0]; + if value < 128 { + return &mut out[..=i]; + } + + out[i] |= 0x80; + value >>= 7; + } + debug_assert_eq!(value, 0); + &mut out[..] + } + + #[inline] + pub fn varint_u64(n: u64, out: &mut [u8; varint_max::()]) -> &mut [u8] { + let mut value = n; + for i in 0..varint_max::() { + out[i] = value.to_le_bytes()[0]; + if value < 128 { + return &mut out[..=i]; + } + + out[i] |= 0x80; + value >>= 7; + } + debug_assert_eq!(value, 0); + &mut out[..] + } + + #[inline] + pub fn varint_u128(n: u128, out: &mut [u8; varint_max::()]) -> &mut [u8] { + let mut value = n; + for i in 0..varint_max::() { + out[i] = value.to_le_bytes()[0]; + if value < 128 { + return &mut out[..=i]; + } + + out[i] |= 0x80; + value >>= 7; + } + debug_assert_eq!(value, 0); + &mut out[..] + } + + pub fn zig_zag_i16(n: i16) -> u16 { + ((n << 1) ^ (n >> 15)) as u16 + } + + pub fn zig_zag_i32(n: i32) -> u32 { + ((n << 1) ^ (n >> 31)) as u32 + } + + pub fn zig_zag_i64(n: i64) -> u64 { + ((n << 1) ^ (n >> 63)) as u64 + } + + pub fn zig_zag_i128(n: i128) -> u128 { + ((n << 1) ^ (n >> 127)) as u128 + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use serde::{Deserialize, Serialize}; + use serde_json::json; + + use crate::to_stdvec_dyn; + use postcard_schema::Schema; + + #[derive(Serialize, Deserialize, Schema)] + struct UnitStruct; + + #[derive(Serialize, Deserialize, Schema)] + struct TupStruct1(u8); + + #[derive(Serialize, Deserialize, Schema)] + struct TupStruct2(u8, u8); + + #[derive(Serialize, Deserialize, Schema)] + struct Struct1 { + pub x: bool, + pub y: u16, + pub z: f64, + } + + #[derive(Serialize, Deserialize, Schema)] + enum Enum1 { + Alpha, + Beta(u8), + Gamma(Vec), + Delta(Struct1), + Epsilon(u8, bool), + // TODO: struct variants are broken in the Schema derive in + // stable postcard, tho it is fixed on the main branch. + // Zeta { a: u8, b: bool }, + } + + #[test] + fn ints() { + let pos = json!(45); + + let t = to_stdvec_dyn(&u8::SCHEMA.into(), &pos).unwrap(); + assert_eq!(t, vec![45]); + + let t = to_stdvec_dyn(&u16::SCHEMA.into(), &pos).unwrap(); + assert_eq!(t, vec![45]); + + let t = to_stdvec_dyn(&u32::SCHEMA.into(), &pos).unwrap(); + assert_eq!(t, vec![45]); + + let t = to_stdvec_dyn(&u64::SCHEMA.into(), &pos).unwrap(); + assert_eq!(t, vec![45]); + + let t = to_stdvec_dyn(&u128::SCHEMA.into(), &pos).unwrap(); + assert_eq!(t, vec![45]); + + let neg = json!(-45); + + let t = to_stdvec_dyn(&i8::SCHEMA.into(), &neg).unwrap(); + // i8s are serialized as-is + assert_eq!(t, vec![211]); + + // All other types get zig-zag'd first + let t = to_stdvec_dyn(&i16::SCHEMA.into(), &neg).unwrap(); + assert_eq!(t, vec![89]); + + let t = to_stdvec_dyn(&i32::SCHEMA.into(), &neg).unwrap(); + assert_eq!(t, vec![89]); + + let t = to_stdvec_dyn(&i64::SCHEMA.into(), &neg).unwrap(); + assert_eq!(t, vec![89]); + + let t = to_stdvec_dyn(&i128::SCHEMA.into(), &neg).unwrap(); + assert_eq!(t, vec![89]); + + let pos = json!(128); + let t = to_stdvec_dyn(&u16::SCHEMA.into(), &pos).unwrap(); + assert_eq!(t, vec![128, 1]); + + let pos = json!(-65); + let t = to_stdvec_dyn(&i16::SCHEMA.into(), &pos).unwrap(); + assert_eq!(t, vec![129, 1]); + } + + #[test] + fn opts() { + let bys = serde_json::to_value(Some(5i16)).unwrap(); + let t = to_stdvec_dyn(&Option::::SCHEMA.into(), &bys).unwrap(); + assert_eq!(t, [0x01, 0x0A]); + let byn = serde_json::to_value(Option::::None).unwrap(); + let t = to_stdvec_dyn(&Option::::SCHEMA.into(), &byn).unwrap(); + assert_eq!(t, [0x00]); + } + + #[test] + fn strs() { + let s = json!("Hello, world!"); + let t = to_stdvec_dyn(&String::SCHEMA.into(), &s).unwrap(); + let mut exp = vec![13]; + exp.extend_from_slice(b"Hello, world!"); + assert_eq!(exp, t); + } + + #[test] + fn seqs() { + let s = json!([1, 2, 3, 4]); + let t = to_stdvec_dyn(&Vec::::SCHEMA.into(), &s).unwrap(); + assert_eq!(vec![4, 1, 2, 3, 4], t); + } + + #[test] + fn tups() { + let byt = serde_json::to_value((true, 5u8, 1.0f32)).unwrap(); + type Tup1 = (bool, u8, f32); + let t = to_stdvec_dyn(&Tup1::SCHEMA.into(), &byt).unwrap(); + assert_eq!(vec![1, 5, 0, 0, 128, 63], t); + } + + #[test] + fn structs() { + let bysct = serde_json::to_value(Struct1 { + x: false, + y: 1000, + z: 4.0, + }) + .unwrap(); + let t = to_stdvec_dyn(&Struct1::SCHEMA.into(), &bysct).unwrap(); + assert_eq!(vec![0, 232, 7, 0, 0, 0, 0, 0, 0, 16, 64], t); + + let bysct = serde_json::to_value(TupStruct1(1)).unwrap(); + let t = to_stdvec_dyn(&TupStruct1::SCHEMA.into(), &bysct).unwrap(); + assert_eq!(vec![1], t); + + let bysct = serde_json::to_value(TupStruct2(1, 2)).unwrap(); + let t = to_stdvec_dyn(&TupStruct2::SCHEMA.into(), &bysct).unwrap(); + assert_eq!(vec![1, 2], t); + + let bysct = serde_json::to_value(UnitStruct).unwrap(); + let t = to_stdvec_dyn(&UnitStruct::SCHEMA.into(), &bysct).unwrap(); + assert_eq!(Vec::::new(), t); + } + + #[test] + fn enums() { + let bye = serde_json::to_value(Enum1::Alpha).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![0], t); + + let bye = serde_json::to_value(Enum1::Beta(4)).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![1, 4], t); + + let bye = serde_json::to_value(Enum1::Gamma(vec![1, 2, 3])).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![2, 3, 1, 2, 3], t); + + let bye = serde_json::to_value(Enum1::Delta(Struct1 { + x: false, + y: 1000, + z: 4.0, + })) + .unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![3, 0, 232, 7, 0, 0, 0, 0, 0, 0, 16, 64], t); + + let bye = serde_json::to_value(Enum1::Epsilon(8, false)).unwrap(); + let t = to_stdvec_dyn(&Enum1::SCHEMA.into(), &bye).unwrap(); + assert_eq!(vec![4, 8, 0], t); + } + + // TODO: we don't implement schema for map types like HashMap, BTreeMap, heapless::IndexMap, or + // similar types. We should fix that. + // + #[test] + fn maps() { + type Map1 = HashMap; + let mut map: Map1 = HashMap::new(); + map.insert("bib".to_string(), 10); + map.insert("bim".to_string(), 20); + map.insert("bap".to_string(), 30); + let bym = serde_json::to_value(map).unwrap(); + let _t = to_stdvec_dyn(&Map1::SCHEMA.into(), &bym).unwrap(); + } + + // Figuring out how serde_json handles various types + #[test] + fn serde_j() { + let bys = serde_json::to_value(Some(5i16)).unwrap(); + assert_eq!(bys.as_i64().unwrap(), 5); + + let byn = serde_json::to_value(Option::::None).unwrap(); + assert!(byn.is_null()); + + let byt = serde_json::to_value((true, 5u8, 1.0f32)).unwrap(); + assert!(byt.is_array()); + + let bysct = serde_json::to_value(Struct1 { + x: false, + y: 1000, + z: 4.0, + }) + .unwrap(); + assert!(bysct.is_object()); + + let bye = serde_json::to_value(Enum1::Alpha).unwrap(); + assert_eq!(bye.as_str().unwrap(), "Alpha"); + let bye = serde_json::to_value(Enum1::Epsilon(1, true)).unwrap(); + assert!(bye.as_object().is_some()); + + let bysct = serde_json::to_value(TupStruct1(1)).unwrap(); + assert!(bysct.is_number()); + + let bysct = serde_json::to_value(TupStruct2(1, 2)).unwrap(); + assert!(bysct.is_array()); + + let bysct = serde_json::to_value(UnitStruct).unwrap(); + assert!(bysct.is_null()); + + // uh oh null coalescing + let byon = serde_json::to_value(Some(())).unwrap(); + assert!(byon.is_null()); + } +}