Attestation processing optimizations for rewards - STF Performance Improvements

Cargo.toml

@@ -56,3 +56,6 @@ quote = "1.0.18"
 clap = { version = "4.5.3", features = ["derive"] }
 convert_case = "0.6.0"
 walkdir = "2.3.3"
+
+[profile.release]
+debug = true

ethereum-consensus/Cargo.toml

@@ -9,8 +9,9 @@ license = "MIT OR Apache-2.0"
 default = ["serde", "async"]
 serde = ["hex", "serde_json", "serde_yaml"]
 async = ["tokio", "tokio-stream"]
-optimized = ["shuffling"]
+optimized = ["shuffling", "attestation-processing"]
 shuffling = [] # supports optimized shuffling routines
+attestation-processing = [] # supports optimized attestation processing
 secret-key-debug = [
 ] # enable if you want to be able to print `crypto::SecretKey`
 spec-tests = [] # enable extra features for testing
@@ -75,3 +76,23 @@ reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "b89af430
 [[bin]]
 name = "ec"
 required-features = ["ec"]
+
+[[example]]
+name = "state_transitions_phase0"
+path = "examples/state_transitions/phase0_blocks.rs"
+
+[[example]]
+name = "state_transitions_altair"
+path = "examples/state_transitions/altair_blocks.rs"
+
+[[example]]
+name = "state_transitions_bellatrix"
+path = "examples/state_transitions/bellatrix_blocks.rs"
+
+[[example]]
+name = "state_transitions_capella"
+path = "examples/state_transitions/capella_blocks.rs"
+
+[[example]]
+name = "state_transitions_deneb"
+path = "examples/state_transitions/deneb_blocks.rs"

ethereum-consensus/examples/state_transitions/altair_blocks.rs

@@ -0,0 +1,58 @@
+use ethereum_consensus::{
+    altair::mainnet as spec,
+    state_transition::mainnet::{Context, Executor},
+    types::{mainnet::SignedBeaconBlock, BeaconState},
+};
+use ssz_rs::prelude::*;
+use std::{error::Error, fs, time::Instant};
+
+fn main() -> std::result::Result<(), Box<dyn Error>> {
+    println!("this example illustrates how the spec applies state transitions to Altair blocks.");
+
+    let state_path =
+        "./ethereum-consensus/examples/mainnet_blocks/altair/prestate/state_2375711.ssz";
+    let f = fs::read(state_path).unwrap();
+    let prestate = spec::BeaconState::deserialize(&f)?;
+    let prestate = BeaconState::Altair(prestate);
+
+    let context = Context::for_mainnet();
+    let mut executor = Executor::new(prestate, context);
+
+    // Read and process blocks at slots 2375712-2375743
+    for slot in 2375712..=2375743 {
+        let block_path = format!(
+            "./ethereum-consensus/examples/mainnet_blocks/altair/beacon_blocks/block_{}.ssz",
+            slot
+        );
+        let block_bytes = fs::read(&block_path)?;
+        if is_skipped_slot(&block_bytes, slot) {
+            continue;
+        }
+        let signed_block = spec::SignedBeaconBlock::deserialize(&block_bytes)?;
+        let block = SignedBeaconBlock::Altair(signed_block);
+
+        let start = Instant::now();
+        executor.apply_block(&block)?;
+        println!("Block at slot {slot} took {:?} to process", start.elapsed());
+    }
+
+    Ok(())
+}
+
+fn is_skipped_slot(block_bytes: &[u8], slot: u64) -> bool {
+    if block_bytes.len() < 100 {
+        match std::str::from_utf8(block_bytes) {
+            Ok(text) if text.contains("NOT_FOUND") => {
+                println!("Slot {} was skipped (no block produced)", slot);
+                println!("\n");
+                true
+            }
+            _ => {
+                println!("Unexpected small file for slot {}", slot);
+                true
+            }
+        }
+    } else {
+        false
+    }
+}

ethereum-consensus/examples/state_transitions/bellatrix_blocks.rs

@@ -0,0 +1,60 @@
+use ethereum_consensus::{
+    bellatrix::mainnet as spec,
+    state_transition::mainnet::{Context, Executor},
+    types::{mainnet::SignedBeaconBlock, BeaconState},
+};
+use ssz_rs::prelude::*;
+use std::{error::Error, fs, time::Instant};
+
+fn main() -> std::result::Result<(), Box<dyn Error>> {
+    println!(
+        "this example illustrates how the spec applies state transitions to Bellatrix blocks."
+    );
+
+    let state_path =
+        "./ethereum-consensus/examples/mainnet_blocks/bellatrix/prestate/state_4636703.ssz";
+    let f = fs::read(state_path).unwrap();
+    let prestate = spec::BeaconState::deserialize(&f)?;
+    let prestate = BeaconState::Bellatrix(prestate);
+
+    let context = Context::for_mainnet();
+    let mut executor = Executor::new(prestate, context);
+
+    // Read and process blocks at slots 4636704-4636735
+    for slot in 4636704..=4636735 {
+        let block_path = format!(
+            "./ethereum-consensus/examples/mainnet_blocks/bellatrix/beacon_blocks/block_{}.ssz",
+            slot
+        );
+        let block_bytes = fs::read(&block_path)?;
+        if is_skipped_slot(&block_bytes, slot) {
+            continue;
+        }
+        let signed_block = spec::SignedBeaconBlock::deserialize(&block_bytes)?;
+        let block = SignedBeaconBlock::Bellatrix(signed_block);
+
+        let start = Instant::now();
+        executor.apply_block(&block)?;
+        println!("Block at slot {slot} took {:?} to process", start.elapsed());
+    }
+
+    Ok(())
+}
+
+fn is_skipped_slot(block_bytes: &[u8], slot: u64) -> bool {
+    if block_bytes.len() < 100 {
+        match std::str::from_utf8(block_bytes) {
+            Ok(text) if text.contains("NOT_FOUND") => {
+                println!("Slot {} was skipped (no block produced)", slot);
+                println!("\n");
+                true
+            }
+            _ => {
+                println!("Unexpected small file for slot {}", slot);
+                true
+            }
+        }
+    } else {
+        false
+    }
+}

ethereum-consensus/examples/state_transitions/capella_blocks.rs

@@ -0,0 +1,58 @@
+use ethereum_consensus::{
+    capella::mainnet as spec,
+    state_transition::mainnet::{Context, Executor},
+    types::{mainnet::SignedBeaconBlock, BeaconState},
+};
+use ssz_rs::prelude::*;
+use std::{error::Error, fs, time::Instant};
+
+fn main() -> std::result::Result<(), Box<dyn Error>> {
+    println!("this example illustrates how the spec applies state transitions to mainnet data.");
+
+    let state_path =
+        "./ethereum-consensus/examples/mainnet_blocks/capella/prestate/state_6209567.ssz";
+    let f = fs::read(state_path).unwrap();
+    let prestate = spec::BeaconState::deserialize(&f)?;
+    let prestate = BeaconState::Capella(prestate);
+
+    let context = Context::for_mainnet();
+    let mut executor = Executor::new(prestate, context);
+
+    // Read and process blocks at slots 6209568-6209599
+    for slot in 6209568..=6209599 {
+        let block_path = format!(
+            "./ethereum-consensus/examples/mainnet_blocks/capella/beacon_blocks/block_{}.ssz",
+            slot
+        );
+        let block_bytes = fs::read(&block_path)?;
+        if is_skipped_slot(&block_bytes, slot) {
+            continue;
+        }
+        let signed_block = spec::SignedBeaconBlock::deserialize(&block_bytes)?;
+        let block = SignedBeaconBlock::Capella(signed_block);
+
+        let start = Instant::now();
+        executor.apply_block(&block)?;
+        println!("Block at slot {slot} took {:?} to process", start.elapsed());
+    }
+
+    Ok(())
+}
+
+fn is_skipped_slot(block_bytes: &[u8], slot: u64) -> bool {
+    if block_bytes.len() < 100 {
+        match std::str::from_utf8(block_bytes) {
+            Ok(text) if text.contains("NOT_FOUND") => {
+                println!("Slot {} was skipped (no block produced)", slot);
+                println!("\n");
+                true
+            }
+            _ => {
+                println!("Unexpected small file for slot {}", slot);
+                true
+            }
+        }
+    } else {
+        false
+    }
+}

ethereum-consensus/examples/state_transitions/deneb_blocks.rs

@@ -0,0 +1,58 @@
+use ethereum_consensus::{
+    deneb::mainnet as spec,
+    state_transition::mainnet::{Context, Executor},
+    types::{mainnet::SignedBeaconBlock, BeaconState},
+};
+use ssz_rs::prelude::*;
+use std::{error::Error, fs, time::Instant};
+
+fn main() -> std::result::Result<(), Box<dyn Error>> {
+    println!("this example illustrates how the spec applies state transitions to mainnet data.");
+
+    let state_path =
+        "./ethereum-consensus/examples/mainnet_blocks/deneb/prestate/state_8626207.ssz";
+    let f = fs::read(state_path).unwrap();
+    let prestate = spec::BeaconState::deserialize(&f)?;
+    let prestate = BeaconState::Deneb(prestate);
+
+    let context = Context::for_mainnet();
+    let mut executor = Executor::new(prestate, context);
+
+    // Read and process blocks at slots 8626208-8626239
+    for slot in 8626208..=8626239 {
+        let block_path = format!(
+            "./ethereum-consensus/examples/mainnet_blocks/deneb/beacon_blocks/block_{}.ssz",
+            slot
+        );
+        let block_bytes = fs::read(&block_path)?;
+        if is_skipped_slot(&block_bytes, slot) {
+            continue;
+        }
+        let signed_block = spec::SignedBeaconBlock::deserialize(&block_bytes)?;
+        let block = SignedBeaconBlock::Deneb(signed_block);
+
+        let start = Instant::now();
+        executor.apply_block(&block)?;
+        println!("Block at slot {slot} took {:?} to process", start.elapsed());
+    }
+
+    Ok(())
+}
+
+fn is_skipped_slot(block_bytes: &[u8], slot: u64) -> bool {
+    if block_bytes.len() < 100 {
+        match std::str::from_utf8(block_bytes) {
+            Ok(text) if text.contains("NOT_FOUND") => {
+                println!("Slot {} was skipped (no block produced)", slot);
+                println!("\n");
+                true
+            }
+            _ => {
+                println!("Unexpected small file for slot {}", slot);
+                true
+            }
+        }
+    } else {
+        false
+    }
+}

ethereum-consensus/examples/state_transitions/phase0_blocks.rs

@@ -0,0 +1,57 @@
+use ethereum_consensus::{
+    phase0::mainnet as spec,
+    state_transition::mainnet::{Context, Executor},
+    types::{mainnet::SignedBeaconBlock, BeaconState},
+};
+use ssz_rs::prelude::*;
+use std::{error::Error, fs, time::Instant};
+
+fn main() -> std::result::Result<(), Box<dyn Error>> {
+    println!("this example illustrates how the spec applies state transitions to Phase0 blocks.");
+
+    let state_path = "./ethereum-consensus/examples/mainnet_blocks/phase0/prestate/state_1999.ssz";
+    let f = fs::read(state_path).unwrap();
+    let prestate = spec::BeaconState::deserialize(&f)?;
+    let prestate = BeaconState::Phase0(prestate);
+
+    let context = Context::for_mainnet();
+    let mut executor = Executor::new(prestate, context);
+
+    // Read and process blocks at slots 2000-2034
+    for slot in 2000..=2034 {
+        let block_path = format!(
+            "./ethereum-consensus/examples/mainnet_blocks/phase0/beacon_blocks/block_{}.ssz",
+            slot
+        );
+        let block_bytes = fs::read(&block_path)?;
+        if is_skipped_slot(&block_bytes, slot) {
+            continue;
+        }
+        let signed_block = spec::SignedBeaconBlock::deserialize(&block_bytes)?;
+        let block = SignedBeaconBlock::Phase0(signed_block);
+
+        let start = Instant::now();
+        executor.apply_block(&block)?;
+        println!("Block at slot {slot} took {:?} to process", start.elapsed());
+    }
+
+    Ok(())
+}
+
+fn is_skipped_slot(block_bytes: &[u8], slot: u64) -> bool {
+    if block_bytes.len() < 100 {
+        match std::str::from_utf8(block_bytes) {
+            Ok(text) if text.contains("NOT_FOUND") => {
+                println!("Slot {} was skipped (no block produced)", slot);
+                println!("\n");
+                true
+            }
+            _ => {
+                println!("Unexpected small file for slot {}", slot);
+                true
+            }
+        }
+    } else {
+        false
+    }
+}