-
Notifications
You must be signed in to change notification settings - Fork 114
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implement .era2 (state snapshots) + import/export from Trin Exe…
…cution
- Loading branch information
Showing
18 changed files
with
893 additions
and
184 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| pub mod memory; | ||
| pub mod stream; | ||
| pub mod types; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| use std::{ | ||
| fs::File, | ||
| io::{Read, Write}, | ||
| path::PathBuf, | ||
| }; | ||
|
|
||
| use super::types::{Entry, Header}; | ||
|
|
||
| /// e2s.rs was built to load full .era/.era2 files into memory and provide a simple API to access the data. | ||
| /// The issue for this is for larger files this wouldn't be feasible, as the entire file would need to be loaded into memory. | ||
| /// This is where e2store_file.rs comes in, it provides a way to read and write e2store files in a streaming fashion. | ||
| pub struct E2StoreStream { | ||
| pub e2store_file: File, | ||
| } | ||
|
|
||
| impl E2StoreStream { | ||
| pub fn new(e2store_path: &PathBuf) -> anyhow::Result<Self> { | ||
| let e2store_file = File::open(e2store_path)?; | ||
| Ok(Self { e2store_file }) | ||
| } | ||
|
|
||
| pub fn next_entry(&mut self) -> anyhow::Result<Entry> { | ||
| let mut buf = vec![0; 8]; | ||
| self.e2store_file.read_exact(&mut buf)?; | ||
| let header = Header::deserialize(&buf)?; | ||
| let mut value = vec![0; header.length as usize]; | ||
| self.e2store_file.read_exact(&mut value)?; | ||
| Ok(Entry { header, value }) | ||
| } | ||
|
|
||
| /// Append an entry to the e2store file. | ||
| pub fn append_entry(&mut self, entry: &Entry) -> anyhow::Result<()> { | ||
| let buf = entry.serialize()?; | ||
| self.e2store_file.write_all(&buf)?; | ||
| Ok(()) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| use anyhow::{anyhow, ensure}; | ||
| use ssz_derive::{Decode, Encode}; | ||
|
|
||
| pub const HEADER_SIZE: u16 = 8; | ||
| const VALUE_SIZE_LIMIT: usize = 1024 * 1024 * 50; // 50 MB | ||
|
|
||
| /// Represents an e2store `Entry` | ||
| #[derive(Default, Debug, Eq, PartialEq, Clone)] | ||
| pub struct Entry { | ||
| pub header: Header, | ||
| pub value: Vec<u8>, | ||
| } | ||
|
|
||
| #[allow(dead_code)] | ||
| impl Entry { | ||
| pub fn new(type_: u16, value: Vec<u8>) -> Self { | ||
| Self { | ||
| header: Header { | ||
| type_, | ||
| length: value.len() as u32, | ||
| reserved: 0, | ||
| }, | ||
| value, | ||
| } | ||
| } | ||
|
|
||
| pub fn length(&self) -> usize { | ||
| HEADER_SIZE as usize + self.header.length as usize | ||
| } | ||
|
|
||
| /// Serialize to a byte vector. | ||
| pub fn serialize(&self) -> anyhow::Result<Vec<u8>> { | ||
| let length = self.length(); | ||
| let mut buf = vec![0; length]; | ||
| self.write(&mut buf)?; | ||
| Ok(buf) | ||
| } | ||
|
|
||
| /// Write to a byte slice. | ||
| pub fn write(&self, buf: &mut [u8]) -> anyhow::Result<()> { | ||
| if self.length() != buf.len() { | ||
| return Err(anyhow!( | ||
| "found invalid buf length for entry: {} - expected {}", | ||
| buf.len(), | ||
| self.length() | ||
| )); | ||
| } | ||
| if self.length() > VALUE_SIZE_LIMIT { | ||
| return Err(anyhow!( | ||
| "entry value size limit exceeded: {} - {}", | ||
| self.length(), | ||
| VALUE_SIZE_LIMIT | ||
| )); | ||
| } | ||
| self.header.write(buf); | ||
| buf[8..].copy_from_slice(&self.value); | ||
| Ok(()) | ||
| } | ||
|
|
||
| /// Deserialize from a byte slice. | ||
| pub fn deserialize(bytes: &[u8]) -> anyhow::Result<Self> { | ||
| let header = Header::deserialize(&bytes[0..8])?; | ||
| if header.length as usize + HEADER_SIZE as usize != bytes.len() { | ||
| return Err(anyhow!( | ||
| "found invalid buf length for entry: {} - expected {}", | ||
| bytes.len(), | ||
| header.length as usize + HEADER_SIZE as usize | ||
| )); | ||
| } | ||
| Ok(Self { | ||
| header: Header::deserialize(&bytes[0..8])?, | ||
| value: bytes[8..].to_vec(), | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| /// Represents the header of an e2store `Entry` | ||
| #[derive(Clone, Debug, Decode, Encode, Default, Eq, PartialEq)] | ||
| pub struct Header { | ||
| pub type_: u16, | ||
| pub length: u32, | ||
| pub reserved: u16, | ||
| } | ||
|
|
||
| impl Header { | ||
| /// Write to a byte slice. | ||
| fn write(&self, buf: &mut [u8]) { | ||
| buf[0..2].copy_from_slice(&self.type_.to_le_bytes()); | ||
| buf[2..6].copy_from_slice(&self.length.to_le_bytes()); | ||
| buf[6..8].copy_from_slice(&self.reserved.to_le_bytes()); | ||
| } | ||
|
|
||
| /// Deserialize from a byte slice. | ||
| pub fn deserialize(bytes: &[u8]) -> anyhow::Result<Self> { | ||
| if bytes.len() != HEADER_SIZE as usize { | ||
| return Err(anyhow!("invalid header size: {}", bytes.len())); | ||
| } | ||
| let type_ = u16::from_le_bytes([bytes[0], bytes[1]]); | ||
| let length = u32::from_le_bytes([bytes[2], bytes[3], bytes[4], bytes[5]]); | ||
| let reserved = u16::from_le_bytes([bytes[6], bytes[7]]); | ||
| if reserved != 0 { | ||
| return Err(anyhow!("invalid reserved value: {} - expected 0", reserved)); | ||
| } | ||
| Ok(Self { | ||
| type_, | ||
| length, | ||
| reserved, | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| #[derive(Clone, Debug, Eq, PartialEq)] | ||
| pub struct VersionEntry { | ||
| version: Entry, | ||
| } | ||
|
|
||
| impl TryFrom<&Entry> for VersionEntry { | ||
| type Error = anyhow::Error; | ||
|
|
||
| fn try_from(entry: &Entry) -> anyhow::Result<Self> { | ||
| ensure!( | ||
| entry.header.type_ == 0x3265, | ||
| "invalid version entry: incorrect header type" | ||
| ); | ||
| ensure!( | ||
| entry.header.length == 0, | ||
| "invalid version entry: incorrect header length" | ||
| ); | ||
| ensure!( | ||
| entry.header.reserved == 0, | ||
| "invalid version entry: incorrect header reserved bytes" | ||
| ); | ||
| ensure!( | ||
| entry.value.is_empty(), | ||
| "invalid version entry: non-empty value" | ||
| ); | ||
| Ok(Self { | ||
| version: entry.clone(), | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| impl TryInto<Entry> for VersionEntry { | ||
| type Error = anyhow::Error; | ||
|
|
||
| fn try_into(self) -> anyhow::Result<Entry> { | ||
| Ok(self.version) | ||
| } | ||
| } |
Oops, something went wrong.