From 2b9dcec93713a6b204b451442a54c7c635de0618 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:10:46 +0100 Subject: [PATCH 1/2] fix: wrong json serialization of ConsensusParams --- proto-compiler/src/constants.rs | 31 ++++++++++-- proto/src/protobuf.rs | 66 +++++++++++++++++++++++--- proto/src/serializers/from_str.rs | 12 +++-- proto/src/serializers/time_duration.rs | 3 ++ proto/src/serializers/timestamp.rs | 8 ++-- proto/tests/unit.rs | 43 ++++++++++++++++- 6 files changed, 145 insertions(+), 18 deletions(-) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index a328e4a2..924e1429 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -91,6 +91,15 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.crypto.Proof", SERIALIZED), (".tendermint.abci.Response.value", DERIVE_FROM), (".tendermint.abci.Request.value", DERIVE_FROM), + // Consensus params + (".tendermint.types.ConsensusParams", SERIALIZED), + (".tendermint.types.ABCIParams", SERIALIZED), + (".tendermint.types.BlockParams", SERIALIZED), + (".tendermint.types.EvidenceParams", SERIALIZED), + (".tendermint.types.ValidatorParams", SERIALIZED), + (".tendermint.types.VersionParams", SERIALIZED), + (".tendermint.types.SynchronyParams", SERIALIZED), + (".tendermint.types.TimeoutParams", SERIALIZED), ]; /// Custom field attributes applied on top of protobuf fields in (a) struct(s) @@ -99,10 +108,6 @@ pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ /// The first item is a path as defined in the prost_build::Config::btree_map /// here: pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[ - ( - ".tendermint.types.EvidenceParams.max_bytes", - QUOTED_WITH_DEFAULT, - ), (".tendermint.version.Consensus.block", QUOTED), (".tendermint.version.Consensus.app", QUOTED_WITH_DEFAULT), (".tendermint.abci.ResponseInfo.data", DEFAULT), @@ -200,4 +205,22 @@ pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.crypto.Proof.total", QUOTED), (".tendermint.crypto.Proof.aunts", VEC_BASE64STRING), (".tendermint.crypto.Proof.leaf_hash", BASE64STRING), + // Consensus params + ( + ".tendermint.types.BlockParams.max_bytes", + QUOTED_WITH_DEFAULT, + ), + (".tendermint.types.BlockParams.max_gas", QUOTED_WITH_DEFAULT), + ( + ".tendermint.types.EvidenceParams.max_age_num_blocks", + QUOTED_WITH_DEFAULT, + ), + ( + ".tendermint.types.EvidenceParams.max_bytes", + QUOTED_WITH_DEFAULT, + ), + ( + ".tendermint.types.VersionParams.app_version", + QUOTED_WITH_DEFAULT, + ), ]; diff --git a/proto/src/protobuf.rs b/proto/src/protobuf.rs index 8cececae..b21e789d 100644 --- a/proto/src/protobuf.rs +++ b/proto/src/protobuf.rs @@ -1,7 +1,12 @@ -// Google protobuf Timestamp and Duration types reimplemented because their comments are turned -// into invalid documentation texts and doctest chokes on them. See https://github.com/danburkert/prost/issues/374 -// Prost does not seem to have a way yet to remove documentations defined in protobuf files. -// These structs are defined in gogoproto v1.3.1 at https://github.com/gogo/protobuf/tree/v1.3.1/protobuf/google/protobuf +// Google protobuf Timestamp and Duration types reimplemented because their +// comments are turned into invalid documentation texts and doctest chokes on them. See https://github.com/danburkert/prost/issues/374 +// Prost does not seem to have a way yet to remove documentations defined in +// protobuf files. These structs are defined in gogoproto v1.3.1 at https://github.com/gogo/protobuf/tree/v1.3.1/protobuf/google/protobuf + +#[cfg(not(feature = "std"))] +use core::fmt; +#[cfg(feature = "std")] +use std::fmt; /// A Timestamp represents a point in time independent of any time zone or local /// calendar, encoded as a count of seconds and fractions of seconds at @@ -17,7 +22,10 @@ /// restricting to that range, we ensure that we can convert to and from [RFC /// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. #[derive(Clone, PartialEq, ::prost::Message, ::serde::Deserialize, ::serde::Serialize)] -#[serde(from = "crate::serializers::timestamp::Rfc3339", into = "crate::serializers::timestamp::Rfc3339")] +#[serde( + from = "crate::serializers::timestamp::Rfc3339", + into = "crate::serializers::timestamp::Rfc3339" +)] pub struct Timestamp { /// Represents seconds of UTC time since Unix epoch /// 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to @@ -38,7 +46,7 @@ pub struct Timestamp { /// or "month". It is related to Timestamp in that the difference between /// two Timestamp values is a Duration and it can be added or subtracted /// from a Timestamp. Range is approximately +-10,000 years. -#[derive(Clone, PartialEq, ::prost::Message, ::serde::Deserialize, ::serde::Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Duration { /// Signed seconds of the span of time. Must be from -315,576,000,000 /// to +315,576,000,000 inclusive. Note: these bounds are computed from: @@ -54,3 +62,49 @@ pub struct Duration { #[prost(int32, tag = "2")] pub nanos: i32, } + +impl serde::Serialize for Duration { + fn serialize(&self, serializer: S) -> Result + where + S: serde::ser::Serializer, + { + let total_nanos = self.seconds * 1_000_000_000 + self.nanos as i64; + serializer.serialize_i64(total_nanos) + } +} + +struct DurationVisitor; + +impl<'de> serde::de::Visitor<'de> for DurationVisitor { + type Value = Duration; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a nanosecond representation of a duration") + } + + fn visit_i128(self, value: i128) -> Result + where + E: serde::de::Error, + { + let seconds = (value / 1_000_000_000) as i64; + let nanos = (value % 1_000_000_000) as i32; + Ok(Duration { seconds, nanos }) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + let value = value.parse::().map_err(E::custom)?; + self.visit_i128(value) + } +} + +impl<'de> serde::Deserialize<'de> for Duration { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + deserializer.deserialize_str(DurationVisitor) + } +} diff --git a/proto/src/serializers/from_str.rs b/proto/src/serializers/from_str.rs index c1762524..0544cc71 100644 --- a/proto/src/serializers/from_str.rs +++ b/proto/src/serializers/from_str.rs @@ -1,16 +1,20 @@ //! Serialize and deserialize any `T` that implements [[core::str::FromStr]] //! and [[core::fmt::Display]] from or into string. Note this can be used for //! all primitive data types. +#[cfg(not(feature = "std"))] +use core::{fmt::Display, str::FromStr}; +#[cfg(feature = "std")] +use std::{fmt::Display, str::FromStr}; + use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer}; use crate::prelude::*; - /// Deserialize string into T pub fn deserialize<'de, D, T>(deserializer: D) -> Result where D: Deserializer<'de>, - T: core::str::FromStr, - ::Err: core::fmt::Display, + T: FromStr, + ::Err: Display, { String::deserialize(deserializer)? .parse::() @@ -21,7 +25,7 @@ where pub fn serialize(value: &T, serializer: S) -> Result where S: Serializer, - T: core::fmt::Display, + T: Display, { format!("{value}").serialize(serializer) } diff --git a/proto/src/serializers/time_duration.rs b/proto/src/serializers/time_duration.rs index b8af3304..f30e943e 100644 --- a/proto/src/serializers/time_duration.rs +++ b/proto/src/serializers/time_duration.rs @@ -1,5 +1,8 @@ //! Serialize/deserialize core::time::Duration type from and into string: +#[cfg(not(feature = "std"))] use core::time::Duration; +#[cfg(feature = "std")] +use std::time::Duration; use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer}; diff --git a/proto/src/serializers/timestamp.rs b/proto/src/serializers/timestamp.rs index fc0a5f3a..945f8813 100644 --- a/proto/src/serializers/timestamp.rs +++ b/proto/src/serializers/timestamp.rs @@ -1,6 +1,8 @@ //! Serialize/deserialize Timestamp type from and into string: - +#[cfg(not(feature = "std"))] use core::fmt::{self, Debug}; +#[cfg(feature = "std")] +use std::fmt::{self, Debug}; use serde::{de::Error as _, ser::Error, Deserialize, Deserializer, Serialize, Serializer}; use time::{ @@ -147,8 +149,8 @@ pub fn to_rfc3339_nanos(t: OffsetDateTime) -> String { /// ie. a RFC3339 date-time with left-padded subsecond digits without /// trailing zeros and no trailing dot. /// -/// [`Display`]: core::fmt::Display -/// [`Debug`]: core::fmt::Debug +/// [`Display`]: fmt::Display +/// [`Debug`]: fmt::Debug pub fn fmt_as_rfc3339_nanos(t: OffsetDateTime, f: &mut impl fmt::Write) -> fmt::Result { let t = t.to_offset(offset!(UTC)); let nanos = t.nanosecond(); diff --git a/proto/tests/unit.rs b/proto/tests/unit.rs index f820160b..bd44a9bf 100644 --- a/proto/tests/unit.rs +++ b/proto/tests/unit.rs @@ -2,7 +2,7 @@ use core::convert::TryFrom; use tenderdash_proto::{ abci::ResponseException, - types::{BlockId as RawBlockId, PartSetHeader as RawPartSetHeader}, + types::{BlockId as RawBlockId, ConsensusParams, PartSetHeader as RawPartSetHeader}, Protobuf, }; @@ -133,3 +133,44 @@ pub fn test_response_exception_from() { "string" ); } + +#[test] +pub fn test_consensus_params_serde() { + let json = r#" + { + "block": { + "max_bytes": "2097152", + "max_gas": "500000000" + }, + "evidence": { + "max_age_num_blocks": "10000", + "max_age_duration": "172800000000000", + "max_bytes": "0" + }, + "validator": { + "pub_key_types": [ + "bls12381" + ] + }, + "version": { + "app_version": "1" + }, + "synchrony": { + "precision": "500000000", + "message_delay": "60000000000" + }, + "timeout": { + "propose": "40000000000", + "propose_delta": "5000000000", + "vote": "40000000000", + "vote_delta": "5000000000", + "bypass_commit_timeout": true + }, + "abci": { + "recheck_tx": true + } + } + "#; + + let _new_params: ConsensusParams = serde_json::from_str(json).unwrap(); +} From 3c4ecad26e88a88508544fef43251ee2053edcfe Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:13:53 +0100 Subject: [PATCH 2/2] chore: fix build --- proto-compiler/src/constants.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/proto-compiler/src/constants.rs b/proto-compiler/src/constants.rs index 924e1429..25f27f3a 100644 --- a/proto-compiler/src/constants.rs +++ b/proto-compiler/src/constants.rs @@ -54,7 +54,6 @@ const DERIVE_FROM_STR: &str = r#"#[derive(derive_more::FromStr)]"#; /// here: pub static CUSTOM_TYPE_ATTRIBUTES: &[(&str, &str)] = &[ (".tendermint.libs.bits.BitArray", SERIALIZED), - (".tendermint.types.EvidenceParams", SERIALIZED), (".tendermint.types.BlockIDFlag", PRIMITIVE_ENUM), (".tendermint.types.Block", SERIALIZED), (".tendermint.types.Data", SERIALIZED),