From 46364824d4363379811e9ece88cb97e37ea54e20 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Fri, 16 Jul 2021 09:51:21 -0500 Subject: [PATCH] Version transaction message and add new message format --- sdk/program/src/message/mod.rs | 28 ++ .../src/{message.rs => message/original.rs} | 26 +- sdk/program/src/message/v0.rs | 385 ++++++++++++++++++ sdk/program/src/message/versions.rs | 262 ++++++++++++ 4 files changed, 678 insertions(+), 23 deletions(-) create mode 100644 sdk/program/src/message/mod.rs rename sdk/program/src/{message.rs => message/original.rs} (96%) create mode 100644 sdk/program/src/message/v0.rs create mode 100644 sdk/program/src/message/versions.rs diff --git a/sdk/program/src/message/mod.rs b/sdk/program/src/message/mod.rs new file mode 100644 index 00000000000000..8a5578025fd5e0 --- /dev/null +++ b/sdk/program/src/message/mod.rs @@ -0,0 +1,28 @@ +//! A library for generating a message from a sequence of instructions + +mod original; +mod v0; +mod versions; + +pub use original::Message; + +pub const MESSAGE_HEADER_LENGTH: usize = 3; + +#[frozen_abi(digest = "BVC5RhetsNpheGipt5rUrkR6RDDUHtD5sCLK1UjymL4S")] +#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)] +#[serde(rename_all = "camelCase")] +pub struct MessageHeader { + /// The number of signatures required for this message to be considered valid. The + /// signatures must match the first `num_required_signatures` of `account_keys`. + /// NOTE: Serialization-related changes must be paired with the direct read at sigverify. + pub num_required_signatures: u8, + + /// The last num_readonly_signed_accounts of the signed keys are read-only accounts. Programs + /// may process multiple transactions that load read-only accounts within a single PoH entry, + /// but are not permitted to credit or debit lamports or modify account data. Transactions + /// targeting the same read-write account are evaluated sequentially. + pub num_readonly_signed_accounts: u8, + + /// The last num_readonly_unsigned_accounts of the unsigned keys are read-only accounts. + pub num_readonly_unsigned_accounts: u8, +} diff --git a/sdk/program/src/message.rs b/sdk/program/src/message/original.rs similarity index 96% rename from sdk/program/src/message.rs rename to sdk/program/src/message/original.rs index 67e360f9da328b..1d2f980be9bb5c 100644 --- a/sdk/program/src/message.rs +++ b/sdk/program/src/message/original.rs @@ -9,6 +9,7 @@ use crate::{ bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable, hash::Hash, instruction::{AccountMeta, CompiledInstruction, Instruction}, + message::MessageHeader, pubkey::Pubkey, short_vec, system_instruction, system_program, sysvar, }; @@ -163,28 +164,7 @@ fn get_program_ids(instructions: &[Instruction]) -> Vec { .collect() } -pub const MESSAGE_HEADER_LENGTH: usize = 3; - -#[frozen_abi(digest = "BVC5RhetsNpheGipt5rUrkR6RDDUHtD5sCLK1UjymL4S")] -#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)] -#[serde(rename_all = "camelCase")] -pub struct MessageHeader { - /// The number of signatures required for this message to be considered valid. The - /// signatures must match the first `num_required_signatures` of `account_keys`. - /// NOTE: Serialization-related changes must be paired with the direct read at sigverify. - pub num_required_signatures: u8, - - /// The last num_readonly_signed_accounts of the signed keys are read-only accounts. Programs - /// may process multiple transactions that load read-only accounts within a single PoH entry, - /// but are not permitted to credit or debit lamports or modify account data. Transactions - /// targeting the same read-write account are evaluated sequentially. - pub num_readonly_signed_accounts: u8, - - /// The last num_readonly_unsigned_accounts of the unsigned keys are read-only accounts. - pub num_readonly_unsigned_accounts: u8, -} - -#[frozen_abi(digest = "BPBJZxpRQ4JS7LGJtsgoyctg4BXyBbbY4uc7FjowtxLV")] +#[frozen_abi(digest = "8Rf1mAQbCgnTLkS4CUK9Tx2i7puRp3aq1qw6AGZCrj1L")] #[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)] #[serde(rename_all = "camelCase")] pub struct Message { @@ -508,7 +488,7 @@ impl Message { #[cfg(test)] mod tests { use super::*; - use crate::{hash, instruction::AccountMeta}; + use crate::{hash, instruction::AccountMeta, message::MESSAGE_HEADER_LENGTH}; use std::collections::HashSet; #[test] diff --git a/sdk/program/src/message/v0.rs b/sdk/program/src/message/v0.rs new file mode 100644 index 00000000000000..1f670b69a95a2e --- /dev/null +++ b/sdk/program/src/message/v0.rs @@ -0,0 +1,385 @@ +#![allow(clippy::integer_arithmetic)] + +use super::MessageHeader; +use crate::{ + hash::Hash, + instruction::CompiledInstruction, + pubkey::Pubkey, + sanitize::{Sanitize, SanitizeError}, + short_vec, +}; + +/// Alternative version of `Message` that supports succinct account loading +/// through an on-chain address map. +#[frozen_abi(digest = "FqrtpvgZZxJBCqcETJ7LYVdaLia2n4izZ813DxgyrW2u")] +#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)] +#[serde(rename_all = "camelCase")] +pub struct Message { + /// The message header, identifying signed and read-only `account_keys` + pub header: MessageHeader, + + /// List of accounts loaded by this transaction. + #[serde(with = "short_vec")] + pub account_keys: Vec, + + /// List of address maps used to succinctly load additional accounts for + /// this transaction. + /// + /// # Notes + /// + /// The last `address_maps.len()` accounts of the read-only unsigned + /// accounts are loaded as address maps. + #[serde(with = "short_vec")] + pub address_maps: Vec, + + /// The blockhash of a recent block. + pub recent_blockhash: Hash, + + /// Instructions that invoke a designated program, are executed in sequence, + /// and committed in one atomic transaction if all succeed. + /// + /// # Notes + /// + /// Account indices will index into the list of addresses constructed from + /// the concatenation of `account_keys` and the `entries` of each address + /// map in sequential order. + #[serde(with = "short_vec")] + pub instructions: Vec, +} + +impl Sanitize for Message { + fn sanitize(&self) -> Result<(), SanitizeError> { + // signing area and read-only non-signing area should not + // overlap + if self.header.num_required_signatures as usize + + self.header.num_readonly_unsigned_accounts as usize + > self.account_keys.len() + { + return Err(SanitizeError::IndexOutOfBounds); + } + + // there should be at least 1 RW fee-payer account. + if self.header.num_readonly_signed_accounts >= self.header.num_required_signatures { + return Err(SanitizeError::IndexOutOfBounds); + } + + // there cannot be more address maps than read-only unsigned accounts. + if self.address_maps.len() > self.header.num_readonly_unsigned_accounts as usize { + return Err(SanitizeError::IndexOutOfBounds); + } + + // each map must load at least one entry + let mut num_loaded_accounts = self.account_keys.len(); + for map in &self.address_maps { + let num_loaded_map_entries = map + .read_only_entries + .len() + .saturating_add(map.writable_entries.len()); + + if num_loaded_map_entries == 0 { + return Err(SanitizeError::InvalidValue); + } + + num_loaded_accounts = num_loaded_accounts.saturating_add(num_loaded_map_entries); + } + + // the number of loaded accounts must be <= 256 since account indices are + // encoded as `u8` + if num_loaded_accounts > 256 { + return Err(SanitizeError::IndexOutOfBounds); + } + + for ci in &self.instructions { + if ci.program_id_index as usize >= num_loaded_accounts { + return Err(SanitizeError::IndexOutOfBounds); + } + // A program cannot be a payer. + if ci.program_id_index == 0 { + return Err(SanitizeError::IndexOutOfBounds); + } + for ai in &ci.accounts { + if *ai as usize >= num_loaded_accounts { + return Err(SanitizeError::IndexOutOfBounds); + } + } + } + + Ok(()) + } +} + +/// Address map specifies which entries to load and +#[frozen_abi(digest = "Bus4bNKecGtgSixqPn3vaaFmzyPVfnmLUwh6as9cZmQa")] +#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)] +#[serde(rename_all = "camelCase")] +pub struct AddressMap { + /// List of map entries to load as read-only. + #[serde(with = "short_vec")] + pub read_only_entries: Vec, + /// List of map entries to load as read-write. + #[serde(with = "short_vec")] + pub writable_entries: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn simple_message() -> Message { + Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + account_keys: vec![Pubkey::new_unique(), Pubkey::new_unique()], + address_maps: vec![AddressMap { + read_only_entries: vec![0], + writable_entries: vec![], + }], + ..Message::default() + } + } + + fn two_map_message() -> Message { + Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 2, + }, + account_keys: vec![ + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + ], + address_maps: vec![ + AddressMap { + read_only_entries: vec![0], + writable_entries: vec![1], + }, + AddressMap { + read_only_entries: vec![1], + writable_entries: vec![0], + }, + ], + ..Message::default() + } + } + + #[test] + fn test_sanitize_account_indices() { + assert!(Message { + account_keys: (0..=u8::MAX).map(|_| Pubkey::new_unique()).collect(), + address_maps: vec![], + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![u8::MAX], + data: vec![], + }], + ..simple_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + account_keys: (0..u8::MAX).map(|_| Pubkey::new_unique()).collect(), + address_maps: vec![], + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![u8::MAX], + data: vec![], + }], + ..simple_message() + } + .sanitize() + .is_err()); + + assert!(Message { + account_keys: (0..u8::MAX).map(|_| Pubkey::new_unique()).collect(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![u8::MAX], + data: vec![], + }], + ..simple_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + account_keys: (0..u8::MAX - 1).map(|_| Pubkey::new_unique()).collect(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![u8::MAX], + data: vec![], + }], + ..simple_message() + } + .sanitize() + .is_err()); + + assert!(Message { + address_maps: vec![ + AddressMap { + read_only_entries: (0..200).step_by(2).collect(), + writable_entries: (1..200).step_by(2).collect(), + }, + AddressMap { + read_only_entries: (0..53).step_by(2).collect(), + writable_entries: (1..53).step_by(2).collect(), + }, + ], + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![u8::MAX], + data: vec![], + }], + ..two_map_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + address_maps: vec![ + AddressMap { + read_only_entries: (0..200).step_by(2).collect(), + writable_entries: (1..200).step_by(2).collect(), + }, + AddressMap { + read_only_entries: (0..52).step_by(2).collect(), + writable_entries: (1..52).step_by(2).collect(), + }, + ], + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![u8::MAX], + data: vec![], + }], + ..two_map_message() + } + .sanitize() + .is_err()); + } + + #[test] + fn test_sanitize_excessive_loaded_accounts() { + assert!(Message { + account_keys: (0..=u8::MAX).map(|_| Pubkey::new_unique()).collect(), + address_maps: vec![], + ..simple_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + account_keys: (0..257).map(|_| Pubkey::new_unique()).collect(), + address_maps: vec![], + ..simple_message() + } + .sanitize() + .is_err()); + + assert!(Message { + account_keys: (0..u8::MAX).map(|_| Pubkey::new_unique()).collect(), + ..simple_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + account_keys: (0..256).map(|_| Pubkey::new_unique()).collect(), + ..simple_message() + } + .sanitize() + .is_err()); + + assert!(Message { + address_maps: vec![ + AddressMap { + read_only_entries: (0..200).step_by(2).collect(), + writable_entries: (1..200).step_by(2).collect(), + }, + AddressMap { + read_only_entries: (0..53).step_by(2).collect(), + writable_entries: (1..53).step_by(2).collect(), + } + ], + ..two_map_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + address_maps: vec![ + AddressMap { + read_only_entries: (0..200).step_by(2).collect(), + writable_entries: (1..200).step_by(2).collect(), + }, + AddressMap { + read_only_entries: (0..200).step_by(2).collect(), + writable_entries: (1..200).step_by(2).collect(), + } + ], + ..two_map_message() + } + .sanitize() + .is_err()); + } + + #[test] + fn test_sanitize_excessive_maps() { + assert!(Message { + header: MessageHeader { + num_readonly_unsigned_accounts: 1, + ..simple_message().header + }, + ..simple_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + header: MessageHeader { + num_readonly_unsigned_accounts: 0, + ..simple_message().header + }, + ..simple_message() + } + .sanitize() + .is_err()); + } + + #[test] + fn test_sanitize_address_map() { + assert!(Message { + address_maps: vec![AddressMap { + read_only_entries: vec![], + writable_entries: vec![0], + }], + ..simple_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + address_maps: vec![AddressMap { + read_only_entries: vec![0], + writable_entries: vec![], + }], + ..simple_message() + } + .sanitize() + .is_ok()); + + assert!(Message { + address_maps: vec![AddressMap { + read_only_entries: vec![], + writable_entries: vec![], + }], + ..simple_message() + } + .sanitize() + .is_err()); + } +} diff --git a/sdk/program/src/message/versions.rs b/sdk/program/src/message/versions.rs new file mode 100644 index 00000000000000..0337a3a5bb7346 --- /dev/null +++ b/sdk/program/src/message/versions.rs @@ -0,0 +1,262 @@ +use { + crate::{ + hash::Hash, + instruction::CompiledInstruction, + message::{v0, Message, MessageHeader}, + pubkey::Pubkey, + sanitize::{Sanitize, SanitizeError}, + short_vec, + }, + serde::{ + de::{self, Deserializer, SeqAccess, Visitor}, + ser::{SerializeTuple, Serializer}, + {Deserialize, Serialize}, + }, + std::fmt, +}; + +/// Bit mask that indicates whether a serialized message is versioned. +const VERSION_PREFIX: u8 = 0x80; + +/// Message versions supported by the Solana runtime. +/// +/// # Serialization +/// +/// If the first bit is set, the remaining 7 bits will be used to determine +/// which message version is serialized starting from version `0`. If the first +/// is bit is not set, all bytes are used to encode the original `Message` +/// format. +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum MessageVersions { + Original(Message), + V0(v0::Message), +} + +impl Sanitize for MessageVersions { + fn sanitize(&self) -> Result<(), SanitizeError> { + match self { + Self::Original(message) => message.sanitize(), + Self::V0(message) => message.sanitize(), + } + } +} + +impl Serialize for MessageVersions { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Self::Original(message) => { + let mut seq = serializer.serialize_tuple(1)?; + seq.serialize_element(message)?; + seq.end() + } + Self::V0(message) => { + let mut seq = serializer.serialize_tuple(2)?; + seq.serialize_element(&VERSION_PREFIX)?; + seq.serialize_element(message)?; + seq.end() + } + } + } +} + +enum MessagePrefix { + Original(u8), + Versioned(u8), +} + +impl<'de> Deserialize<'de> for MessagePrefix { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct PrefixVisitor; + + impl<'de> Visitor<'de> for PrefixVisitor { + type Value = MessagePrefix; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("message prefix byte") + } + + fn visit_u8(self, byte: u8) -> Result { + if byte & VERSION_PREFIX != 0 { + Ok(MessagePrefix::Versioned(byte ^ VERSION_PREFIX)) + } else { + Ok(MessagePrefix::Original(byte)) + } + } + } + + deserializer.deserialize_u8(PrefixVisitor) + } +} + +impl<'de> Deserialize<'de> for MessageVersions { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct MessageVisitor; + + impl<'de> Visitor<'de> for MessageVisitor { + type Value = MessageVersions; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("message bytes") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let prefix: MessagePrefix = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + + match prefix { + MessagePrefix::Original(num_required_signatures) => { + // The remaining fields of the original Message struct after the first byte. + #[derive(Serialize, Deserialize)] + struct RemainingMessage { + pub num_readonly_signed_accounts: u8, + pub num_readonly_unsigned_accounts: u8, + #[serde(with = "short_vec")] + pub account_keys: Vec, + pub recent_blockhash: Hash, + #[serde(with = "short_vec")] + pub instructions: Vec, + } + + let rm: RemainingMessage = seq.next_element()?.ok_or_else(|| { + // will never happen since tuple length is always 2 + de::Error::invalid_length(1, &self) + })?; + + Ok(MessageVersions::Original(Message { + header: MessageHeader { + num_required_signatures, + num_readonly_signed_accounts: rm.num_readonly_signed_accounts, + num_readonly_unsigned_accounts: rm.num_readonly_unsigned_accounts, + }, + account_keys: rm.account_keys, + recent_blockhash: rm.recent_blockhash, + instructions: rm.instructions, + })) + } + MessagePrefix::Versioned(version) => { + if version == 0 { + Ok(MessageVersions::V0(seq.next_element()?.ok_or_else( + || { + // will never happen since tuple length is always 2 + de::Error::invalid_length(1, &self) + }, + )?)) + } else { + Err(de::Error::invalid_value( + de::Unexpected::Unsigned(version as u64), + &"supported versions: [0]", + )) + } + } + } + } + } + + deserializer.deserialize_tuple(2, MessageVisitor) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + instruction::{AccountMeta, Instruction}, + message::v0::AddressMap, + }; + + #[test] + fn test_original_message_serialization() { + let program_id0 = Pubkey::new_unique(); + let program_id1 = Pubkey::new_unique(); + let id0 = Pubkey::new_unique(); + let id1 = Pubkey::new_unique(); + let id2 = Pubkey::new_unique(); + let id3 = Pubkey::new_unique(); + let instructions = vec![ + Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id0, false)]), + Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id1, true)]), + Instruction::new_with_bincode( + program_id1, + &0, + vec![AccountMeta::new_readonly(id2, false)], + ), + Instruction::new_with_bincode( + program_id1, + &0, + vec![AccountMeta::new_readonly(id3, true)], + ), + ]; + + let mut message = Message::new(&instructions, Some(&id1)); + message.recent_blockhash = Hash::new_unique(); + + let bytes1 = bincode::serialize(&message).unwrap(); + let bytes2 = bincode::serialize(&MessageVersions::Original(message.clone())).unwrap(); + + assert_eq!(bytes1, bytes2); + + let message1: Message = bincode::deserialize(&bytes1).unwrap(); + let message2: MessageVersions = bincode::deserialize(&bytes2).unwrap(); + + if let MessageVersions::Original(message2) = message2 { + assert_eq!(message, message1); + assert_eq!(message1, message2); + } else { + panic!("should deserialize to original message"); + } + } + + #[test] + fn test_versioned_message_serialization() { + let message = v0::Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 2, + }, + recent_blockhash: Hash::new_unique(), + account_keys: vec![ + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + ], + address_maps: vec![ + AddressMap { + read_only_entries: vec![0], + writable_entries: vec![1], + }, + AddressMap { + read_only_entries: vec![1], + writable_entries: vec![0], + }, + ], + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }], + }; + + let bytes = bincode::serialize(&MessageVersions::V0(message.clone())).unwrap(); + let message_from_bytes: MessageVersions = bincode::deserialize(&bytes).unwrap(); + + if let MessageVersions::V0(message_from_bytes) = message_from_bytes { + assert_eq!(message, message_from_bytes); + } else { + panic!("should deserialize to versioned message"); + } + } +}