[WEEKLY RELEASE] HotShot 0.5.56 (#1535)

Closes #<ISSUE_NUMBER>
<!-- These comments should help create a useful PR message, please
delete any remaining comments before opening the PR. -->
<!-- If there is no issue number make sure to describe clearly *why*
this PR is necessary. -->
<!-- Mention open questions, remaining TODOs, if any -->

### This PR:
<!-- Describe what this PR adds to this repo and why -->
<!-- E.g. -->
<!-- * Implements feature 1 -->
<!-- * Fixes bug 3 -->

### This PR does not:
<!-- Describe what is out of scope for this PR, if applicable. Leave
this section blank if it's not applicable -->
<!-- This section helps avoid the reviewer having to needlessly point
out missing parts -->
<!-- * Implement feature 3 because that feature is blocked by Issue 4
-->
<!-- * Implement xyz because that is tracked in issue #123. -->
<!-- * Address xzy for which I opened issue #456 -->

### Key places to review:
<!-- Describe key places for reviewers to pay close attention to -->
<!-- * file.rs, `add_integers` function -->
<!-- Or directly comment on those files/lines to make it easier for the
reviewers -->

<!-- ### How to test this PR:  -->
<!-- Optional, uncomment the above line if this is relevant to your PR
-->
<!-- If your PR is fully tested through CI there is no need to add this
section -->
<!-- * E.g. `just test` -->

<!-- ### Things tested -->
<!-- Anything that was manually tested (that is not tested in CI). -->
<!-- E.g. building/running of docker containers. Changes to docker demo,
... -->
<!-- Especially mention anything untested, with reasoning and link an
issue to resolve this. -->

<!-- Complete the following items before creating this PR -->
<!-- [ ] Issue linked or PR description mentions why this change is
necessary. -->
<!-- [ ] PR description is clear enough for reviewers. -->
<!-- [ ] Documentation for changes (additions) has been updated (added).
-->
<!-- [ ] If this is a draft it is marked as "draft".  -->

<!-- To make changes to this template edit
https://github.com/EspressoSystems/.github/blob/main/PULL_REQUEST_TEMPLATE.md
-->

Cargo.toml

@@ -47,19 +47,19 @@ dotenvy = "0.15"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.55" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.23" }
-hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.23" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.29" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.56" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.25" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.25" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.30" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" }
+hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
 
 # Push CDN imports
 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [
@@ -99,9 +99,9 @@ jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4
 jf-utils = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5" }
 snafu = "0.8"
 strum = { version = "0.26", features = ["derive"] }
-surf-disco = "0.6"
+surf-disco = "0.7"
 tagged-base64 = "0.4"
-tide-disco = "0.6"
+tide-disco = "0.7"
 time = "0.3"
 tracing = "0.1"
 bytesize = "1.3"

builder/src/bin/permissionless-builder.rs

@@ -3,6 +3,7 @@ use builder::non_permissioned::{build_instance_state, BuilderConfig};
 use clap::Parser;
 use cld::ClDuration;
 use es_version::SEQUENCER_VERSION;
+use hotshot::traits::ValidatedState;
 use hotshot_types::data::ViewNumber;
 use hotshot_types::traits::node_implementation::ConsensusTime;
 use sequencer::{eth_signature_key::EthKeyPair, Genesis, L1Params};
@@ -127,6 +128,7 @@ async fn main() -> anyhow::Result<()> {
     let txn_timeout_duration = api_response_timeout_duration / 4;
 
     let buffer_view_num_count = opt.buffer_view_num_count;
+    let validated_state = ValidatedState::genesis(&instance_state).0;
 
     let _builder_config = BuilderConfig::init(
         builder_key_pair,
@@ -139,6 +141,7 @@ async fn main() -> anyhow::Result<()> {
         api_response_timeout_duration,
         buffer_view_num_count,
         txn_timeout_duration,
+        validated_state,
     )
     .await;
 

builder/src/lib.rs

@@ -322,7 +322,7 @@ pub mod testing {
             bind_version: Ver,
             options: impl PersistenceOptions<Persistence = P>,
         ) -> Vec<(
-            SystemContextHandle<SeqTypes, Node<network::Memory, P>>,
+            Arc<SystemContextHandle<SeqTypes, Node<network::Memory, P>>>,
             Option<StateSigner<Ver>>,
         )> {
             let num_staked_nodes = self.num_staked_nodes();
@@ -348,7 +348,7 @@ pub mod testing {
                         )
                         .await;
                     // wrapped in some because need to take later
-                    (hotshot_handle, Some(state_signer))
+                    (Arc::new(hotshot_handle), Some(state_signer))
                 }
             }))
             .await
@@ -452,7 +452,7 @@ pub mod testing {
             hotshot_events_api_url: Url,
             known_nodes_with_stake: Vec<PeerConfig<VerKey>>,
             num_non_staking_nodes: usize,
-            hotshot_context_handle: SystemContextHandle<SeqTypes, Node<network::Memory, P>>,
+            hotshot_context_handle: Arc<SystemContextHandle<SeqTypes, Node<network::Memory, P>>>,
         ) {
             // create a event streamer
             let events_streamer = Arc::new(RwLock::new(EventsStreamer::new(
@@ -560,6 +560,7 @@ pub mod testing {
                 Duration::from_millis(2000),
                 15,
                 Duration::from_millis(500),
+                ValidatedState::default(),
             )
             .await
             .unwrap();
@@ -584,7 +585,7 @@ pub mod testing {
     {
         pub async fn init_permissioned_builder(
             hotshot_test_config: HotShotTestConfig,
-            hotshot_handle: SystemContextHandle<SeqTypes, Node<network::Memory, P>>,
+            hotshot_handle: Arc<SystemContextHandle<SeqTypes, Node<network::Memory, P>>>,
             node_id: u64,
             state_signer: StateSigner<Ver>,
             hotshot_builder_api_url: Url,
@@ -612,7 +613,7 @@ pub mod testing {
             let bootstrapped_view = ViewNumber::new(0);
 
             let builder_context = BuilderContext::init(
-                hotshot_handle,
+                Arc::clone(&hotshot_handle),
                 state_signer,
                 node_id,
                 key_pair,
@@ -623,6 +624,7 @@ pub mod testing {
                 Duration::from_millis(2000),
                 15,
                 Duration::from_millis(500),
+                ValidatedState::default(),
             )
             .await
             .unwrap();
@@ -704,7 +706,8 @@ mod test {
         let mut parent = {
             // TODO refactor repeated code from other tests
             let (genesis_payload, genesis_ns_table) =
-                Payload::from_transactions([], &genesis_state)
+                Payload::from_transactions([], &ValidatedState::default(), &genesis_state)
+                    .await
                     .expect("unable to create genesis payload");
             let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table);
             let genesis_commitment = {

builder/src/non_permissioned.rs

@@ -36,7 +36,7 @@ use hotshot_types::{
 };
 use sequencer::{
     catchup::StatePeers, eth_signature_key::EthKeyPair, l1_client::L1Client, ChainConfig, L1Params,
-    NetworkParams, NodeState, Payload, PrivKey, PubKey, SeqTypes,
+    NetworkParams, NodeState, Payload, PrivKey, PubKey, SeqTypes, ValidatedState,
 };
 
 use hotshot_events_service::{
@@ -87,6 +87,7 @@ impl BuilderConfig {
         max_api_timeout_duration: Duration,
         buffered_view_num_count: usize,
         maximize_txns_count_timeout_duration: Duration,
+        validated_state: ValidatedState,
     ) -> anyhow::Result<Self> {
         tracing::info!(
             address = %builder_key_pair.fee_account(),
@@ -114,8 +115,10 @@ impl BuilderConfig {
         // builder api request channel
         let (req_sender, req_receiver) = broadcast::<MessageType<SeqTypes>>(channel_capacity.get());
 
-        let (genesis_payload, genesis_ns_table) = Payload::from_transactions([], &instance_state)
-            .expect("genesis payload construction failed");
+        let (genesis_payload, genesis_ns_table) =
+            Payload::from_transactions([], &validated_state, &instance_state)
+                .await
+                .expect("genesis payload construction failed");
 
         let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table);
 
@@ -159,6 +162,8 @@ impl BuilderConfig {
                 .as_u64()
                 .context("the base fee exceeds the maximum amount that a builder can pay (defined by u64::MAX)")?,
             Arc::new(instance_state),
+            Duration::from_secs(60),
+            Arc::new(validated_state),
         );
 
         // spawn the builder event loop
@@ -274,10 +279,8 @@ mod test {
         let num_non_staking_nodes = hotshot_config.config.num_nodes_without_stake;
 
         // non-staking node handle
-        let hotshot_context_handle = handles
-            [NonPermissionedBuilderTestConfig::SUBSCRIBED_DA_NODE_ID]
-            .0
-            .clone();
+        let hotshot_context_handle =
+            &handles[NonPermissionedBuilderTestConfig::SUBSCRIBED_DA_NODE_ID].0;
 
         // hotshot event streaming api url
         let hotshot_events_streaming_api_url = HotShotTestConfig::hotshot_event_streaming_api_url();
@@ -287,7 +290,7 @@ mod test {
             hotshot_events_streaming_api_url.clone(),
             known_nodes_with_stake,
             num_non_staking_nodes,
-            hotshot_context_handle,
+            Arc::clone(hotshot_context_handle),
         );
 
         // builder api url

builder/src/permissioned.rs

@@ -108,7 +108,7 @@ pub struct BuilderContext<
     Ver: StaticVersionType + 'static,
 > {
     /// The consensus handle
-    pub hotshot_handle: Consensus<N, P>,
+    pub hotshot_handle: Arc<Consensus<N, P>>,
 
     /// Index of this sequencer node
     pub node_index: u64,
@@ -186,7 +186,7 @@ pub async fn init_node<P: SequencerPersistence, Ver: StaticVersionType + 'static
     // Initialize the push CDN network (and perform the initial connection)
     let cdn_network = PushCdnNetwork::new(
         network_params.cdn_endpoint,
-        vec![Topic::Global, Topic::DA],
+        vec![Topic::Global, Topic::Da],
         KeyPair {
             public_key: WrappedSignatureKey(my_config.public_key),
             private_key: my_config.private_key.clone(),
@@ -259,7 +259,7 @@ pub async fn init_node<P: SequencerPersistence, Ver: StaticVersionType + 'static
         chain_config: genesis.chain_config,
         l1_client,
         genesis_header: genesis.header,
-        genesis_state,
+        genesis_state: genesis_state.clone(),
         l1_genesis,
         peers: Arc::new(StatePeers::<Ver>::from_urls(network_params.state_peers)),
         node_id: node_index,
@@ -283,7 +283,7 @@ pub async fn init_node<P: SequencerPersistence, Ver: StaticVersionType + 'static
     .await;
 
     let ctx = BuilderContext::init(
-        hotshot_handle,
+        Arc::new(hotshot_handle),
         state_signer,
         node_index,
         eth_key_pair,
@@ -294,6 +294,7 @@ pub async fn init_node<P: SequencerPersistence, Ver: StaticVersionType + 'static
         max_api_timeout_duration,
         buffered_view_num_count,
         maximize_txns_count_timeout_duration,
+        genesis_state,
     )
     .await?;
 
@@ -354,7 +355,9 @@ pub async fn init_hotshot<
         config,
         memberships,
         networks,
-        HotShotInitializer::from_genesis(instance_state).unwrap(),
+        HotShotInitializer::from_genesis(instance_state)
+            .await
+            .unwrap(),
         ConsensusMetricsValue::new(metrics),
         da_storage,
     )
@@ -378,7 +381,7 @@ impl<N: network::Type, P: SequencerPersistence, Ver: StaticVersionType + 'static
     /// Constructor
     #[allow(clippy::too_many_arguments)]
     pub async fn init(
-        hotshot_handle: Consensus<N, P>,
+        hotshot_handle: Arc<Consensus<N, P>>,
         state_signer: StateSigner<Ver>,
         node_index: u64,
         eth_key_pair: EthKeyPair,
@@ -389,6 +392,7 @@ impl<N: network::Type, P: SequencerPersistence, Ver: StaticVersionType + 'static
         max_api_timeout_duration: Duration,
         buffered_view_num_count: usize,
         maximize_txns_count_timeout_duration: Duration,
+        validated_state: ValidatedState,
     ) -> anyhow::Result<Self> {
         // tx channel
         let (tx_sender, tx_receiver) = broadcast::<MessageType<SeqTypes>>(channel_capacity.get());
@@ -406,8 +410,10 @@ impl<N: network::Type, P: SequencerPersistence, Ver: StaticVersionType + 'static
         // builder api request channel
         let (req_sender, req_receiver) = broadcast::<MessageType<SeqTypes>>(channel_capacity.get());
 
-        let (genesis_payload, genesis_ns_table) = Payload::from_transactions([], &instance_state)
-            .expect("genesis payload construction failed");
+        let (genesis_payload, genesis_ns_table) =
+            Payload::from_transactions([], &validated_state, &instance_state)
+                .await
+                .expect("genesis payload construction failed");
 
         let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table);
 
@@ -452,17 +458,19 @@ impl<N: network::Type, P: SequencerPersistence, Ver: StaticVersionType + 'static
                 .as_u64()
                 .context("the base fee exceeds the maximum amount that a builder can pay (defined by u64::MAX)")?,
             Arc::new(instance_state),
+            Duration::from_secs(60),
+            Arc::new(validated_state),
         );
 
-        let hotshot_handle_clone = hotshot_handle.clone();
+        let hotshot_handle_clone = Arc::clone(&hotshot_handle);
         // spawn the builder service
         async_spawn(async move {
             run_permissioned_standalone_builder_service(
                 tx_sender,
                 da_sender,
                 qc_sender,
                 decide_sender,
-                hotshot_handle,
+                hotshot_handle_clone,
             )
             .await;
         });
@@ -483,7 +491,7 @@ impl<N: network::Type, P: SequencerPersistence, Ver: StaticVersionType + 'static
         run_builder_api_service(hotshot_builder_api_url.clone(), proxy_global_state);
 
         let ctx = Self {
-            hotshot_handle: hotshot_handle_clone,
+            hotshot_handle: Arc::clone(&hotshot_handle),
             node_index,
             state_signer: Arc::new(state_signer),
             wait_for_orchestrator: None,
@@ -569,7 +577,7 @@ mod test {
 
         let node_id = total_nodes - 1;
         // non-staking node handle
-        let hotshot_context_handle = handles[node_id].0.clone();
+        let hotshot_context_handle = Arc::clone(&handles[node_id].0);
         let state_signer = handles[node_id].1.take().unwrap();
 
         // builder api url