Update simple builder for marketplace (#3620)

crates/examples/infra/mod.rs

@@ -408,7 +408,7 @@ pub trait RunDa<
         let marketplace_config = MarketplaceConfig {
             auction_results_provider: TestAuctionResultsProvider::<TYPES>::default().into(),
             // TODO: we need to pass a valid fallback builder url here somehow
-            fallback_builder_url: url::Url::parse("http://localhost").unwrap(),
+            fallback_builder_url: config.config.builder_urls.first().clone(),
         };
 
         SystemContext::init(

crates/hotshot/src/traits/networking/libp2p_network.rs

@@ -74,7 +74,6 @@ use libp2p_networking::{
     },
     reexport::{Multiaddr, ResponseChannel},
 };
-
 use rand::{rngs::StdRng, seq::IteratorRandom, SeedableRng};
 use serde::Serialize;
 use tracing::{debug, error, info, instrument, trace, warn};

crates/task-impls/src/builder.rs

@@ -12,6 +12,7 @@ use hotshot_builder_api::v0_1::{
     builder::{BuildError, Error as BuilderApiError},
 };
 use hotshot_types::{
+    constants::LEGACY_BUILDER_MODULE,
     traits::{node_implementation::NodeType, signature_key::SignatureKey},
     vid::VidCommitment,
 };
@@ -64,8 +65,8 @@ impl From<BuilderApiError> for BuilderClientError {
 
 /// Client for builder API
 pub struct BuilderClient<TYPES: NodeType, Ver: StaticVersionType> {
-    /// Underlying surf_disco::Client
-    inner: Client<BuilderApiError, Ver>,
+    /// Underlying surf_disco::Client for the legacy builder api
+    client: Client<BuilderApiError, Ver>,
     /// Marker for [`NodeType`] used here
     _marker: std::marker::PhantomData<TYPES>,
 }
@@ -77,8 +78,10 @@ impl<TYPES: NodeType, Ver: StaticVersionType> BuilderClient<TYPES, Ver> {
     ///
     /// If the URL is malformed.
     pub fn new(base_url: impl Into<Url>) -> Self {
+        let url = base_url.into();
+
         Self {
-            inner: Client::builder(base_url.into().join("block_info").unwrap())
+            client: Client::builder(url.clone())
                 .set_timeout(Some(Duration::from_secs(2)))
                 .build(),
             _marker: std::marker::PhantomData,
@@ -93,7 +96,7 @@ impl<TYPES: NodeType, Ver: StaticVersionType> BuilderClient<TYPES, Ver> {
         let mut backoff = Duration::from_millis(50);
         while Instant::now() < timeout {
             if matches!(
-                self.inner.healthcheck::<HealthStatus>().await,
+                self.client.healthcheck::<HealthStatus>().await,
                 Ok(HealthStatus::Available)
             ) {
                 return true;
@@ -117,9 +120,9 @@ impl<TYPES: NodeType, Ver: StaticVersionType> BuilderClient<TYPES, Ver> {
         signature: &<<TYPES as NodeType>::SignatureKey as SignatureKey>::PureAssembledSignatureType,
     ) -> Result<Vec<AvailableBlockInfo<TYPES>>, BuilderClientError> {
         let encoded_signature: TaggedBase64 = signature.clone().into();
-        self.inner
+        self.client
             .get(&format!(
-                "availableblocks/{parent}/{view_number}/{sender}/{encoded_signature}"
+                "{LEGACY_BUILDER_MODULE}/availableblocks/{parent}/{view_number}/{sender}/{encoded_signature}"
             ))
             .send()
             .await
@@ -129,10 +132,10 @@ impl<TYPES: NodeType, Ver: StaticVersionType> BuilderClient<TYPES, Ver> {
 
 /// Version 0.1
 pub mod v0_1 {
-
     use hotshot_builder_api::v0_1::block_info::{AvailableBlockData, AvailableBlockHeaderInput};
     pub use hotshot_builder_api::v0_1::Version;
     use hotshot_types::{
+        constants::LEGACY_BUILDER_MODULE,
         traits::{node_implementation::NodeType, signature_key::SignatureKey},
         utils::BuilderCommitment,
     };
@@ -157,9 +160,9 @@ pub mod v0_1 {
             signature: &<<TYPES as NodeType>::SignatureKey as SignatureKey>::PureAssembledSignatureType,
         ) -> Result<AvailableBlockHeaderInput<TYPES>, BuilderClientError> {
             let encoded_signature: TaggedBase64 = signature.clone().into();
-            self.inner
+            self.client
                 .get(&format!(
-                    "claimheaderinput/{block_hash}/{view_number}/{sender}/{encoded_signature}"
+                    "{LEGACY_BUILDER_MODULE}/claimheaderinput/{block_hash}/{view_number}/{sender}/{encoded_signature}"
                 ))
                 .send()
                 .await
@@ -179,9 +182,9 @@ pub mod v0_1 {
             signature: &<<TYPES as NodeType>::SignatureKey as SignatureKey>::PureAssembledSignatureType,
         ) -> Result<AvailableBlockData<TYPES>, BuilderClientError> {
             let encoded_signature: TaggedBase64 = signature.clone().into();
-            self.inner
+            self.client
                 .get(&format!(
-                    "claimblock/{block_hash}/{view_number}/{sender}/{encoded_signature}"
+                    "{LEGACY_BUILDER_MODULE}/claimblock/{block_hash}/{view_number}/{sender}/{encoded_signature}"
                 ))
                 .send()
                 .await
@@ -204,7 +207,8 @@ pub mod v0_2 {
 pub mod v0_3 {
     pub use hotshot_builder_api::v0_3::Version;
     use hotshot_types::{
-        bundle::Bundle, traits::node_implementation::NodeType, vid::VidCommitment,
+        bundle::Bundle, constants::MARKETPLACE_BUILDER_MODULE,
+        traits::node_implementation::NodeType, vid::VidCommitment,
     };
     use vbs::version::StaticVersion;
 
@@ -225,8 +229,10 @@ pub mod v0_3 {
             parent_hash: VidCommitment,
             view_number: u64,
         ) -> Result<Bundle<TYPES>, BuilderClientError> {
-            self.inner
-                .get(&format!("bundle/{parent_view}/{parent_hash}/{view_number}"))
+            self.client
+                .get(&format!(
+                    "{MARKETPLACE_BUILDER_MODULE}/bundle/{parent_view}/{parent_hash}/{view_number}"
+                ))
                 .send()
                 .await
                 .map_err(Into::into)

crates/testing/src/block_builder/mod.rs

@@ -11,15 +11,21 @@ use async_compatibility_layer::art::async_spawn;
 use async_trait::async_trait;
 use futures::Stream;
 use hotshot::{traits::BlockPayload, types::Event};
-use hotshot_builder_api::v0_1::{
-    block_info::{AvailableBlockData, AvailableBlockHeaderInput, AvailableBlockInfo},
-    builder::{Error, Options},
-    data_source::BuilderDataSource,
+use hotshot_builder_api::{
+    v0_1,
+    v0_1::{
+        block_info::{AvailableBlockData, AvailableBlockHeaderInput, AvailableBlockInfo},
+        builder::{Error, Options},
+    },
+    v0_3,
 };
-use hotshot_types::traits::{
-    block_contents::{precompute_vid_commitment, EncodeBytes},
-    node_implementation::NodeType,
-    signature_key::BuilderSignatureKey,
+use hotshot_types::{
+    constants::{LEGACY_BUILDER_MODULE, MARKETPLACE_BUILDER_MODULE},
+    traits::{
+        block_contents::{precompute_vid_commitment, EncodeBytes},
+        node_implementation::NodeType,
+        signature_key::BuilderSignatureKey,
+    },
 };
 use tide_disco::{method::ReadState, App, Url};
 use vbs::version::StaticVersionType;
@@ -62,7 +68,7 @@ struct BlockEntry<TYPES: NodeType> {
     header_input: Option<AvailableBlockHeaderInput<TYPES>>,
 }
 
-/// Construct a tide disco app that mocks the builder API.
+/// Construct a tide disco app that mocks the builder API 0.1 + 0.3.
 ///
 /// # Panics
 /// If constructing and launching the builder fails for any reason
@@ -74,7 +80,63 @@ pub fn run_builder_source<TYPES, Source>(
     TYPES: NodeType,
     <TYPES as NodeType>::InstanceState: Default,
     Source: Clone + Send + Sync + tide_disco::method::ReadState + 'static,
-    <Source as ReadState>::State: Sync + Send + BuilderDataSource<TYPES>,
+    <Source as ReadState>::State: Sync
+        + Send
+        + v0_1::data_source::BuilderDataSource<TYPES>
+        + v0_3::data_source::BuilderDataSource<TYPES>,
+{
+    async_spawn(async move {
+        let start_builder = |url: Url, source: Source| -> _ {
+            let builder_api_0_1 = hotshot_builder_api::v0_1::builder::define_api::<Source, TYPES>(
+                &Options::default(),
+            )
+            .expect("Failed to construct the builder API");
+            let builder_api_0_3 = hotshot_builder_api::v0_3::builder::define_api::<Source, TYPES>(
+                &Options::default(),
+            )
+            .expect("Failed to construct the builder API");
+            let mut app: App<Source, Error> = App::with_state(source);
+            app.register_module(LEGACY_BUILDER_MODULE, builder_api_0_1)
+                .expect("Failed to register the builder API 0.1")
+                .register_module(MARKETPLACE_BUILDER_MODULE, builder_api_0_3)
+                .expect("Failed to register the builder API 0.3");
+            async_spawn(app.serve(url, hotshot_builder_api::v0_1::Version::instance()))
+        };
+
+        let mut handle = Some(start_builder(url.clone(), source.clone()));
+
+        while let Ok(event) = change_receiver.recv().await {
+            match event {
+                BuilderChange::Up if handle.is_none() => {
+                    handle = Some(start_builder(url.clone(), source.clone()));
+                }
+                BuilderChange::Down => {
+                    if let Some(handle) = handle.take() {
+                        #[cfg(async_executor_impl = "tokio")]
+                        handle.abort();
+                        #[cfg(async_executor_impl = "async-std")]
+                        handle.cancel().await;
+                    }
+                }
+                _ => {}
+            }
+        }
+    });
+}
+
+/// Construct a tide disco app that mocks the builder API 0.1.
+///
+/// # Panics
+/// If constructing and launching the builder fails for any reason
+pub fn run_builder_source_0_1<TYPES, Source>(
+    url: Url,
+    mut change_receiver: Receiver<BuilderChange>,
+    source: Source,
+) where
+    TYPES: NodeType,
+    <TYPES as NodeType>::InstanceState: Default,
+    Source: Clone + Send + Sync + tide_disco::method::ReadState + 'static,
+    <Source as ReadState>::State: Sync + Send + v0_1::data_source::BuilderDataSource<TYPES>,
 {
     async_spawn(async move {
         let start_builder = |url: Url, source: Source| -> _ {
@@ -83,7 +145,7 @@ pub fn run_builder_source<TYPES, Source>(
             )
             .expect("Failed to construct the builder API");
             let mut app: App<Source, Error> = App::with_state(source);
-            app.register_module("block_info", builder_api)
+            app.register_module(LEGACY_BUILDER_MODULE, builder_api)
                 .expect("Failed to register the builder API");
             async_spawn(app.serve(url, hotshot_builder_api::v0_1::Version::instance()))
         };

crates/testing/src/block_builder/random.rs

@@ -37,7 +37,9 @@ use lru::LruCache;
 use rand::{rngs::SmallRng, Rng, RngCore, SeedableRng};
 use tide_disco::{method::ReadState, Url};
 
-use super::{build_block, run_builder_source, BlockEntry, BuilderTask, TestBuilderImplementation};
+use super::{
+    build_block, run_builder_source_0_1, BlockEntry, BuilderTask, TestBuilderImplementation,
+};
 use crate::test_builder::BuilderChange;
 
 pub struct RandomBuilderImplementation;
@@ -90,7 +92,7 @@ where
         let (change_sender, change_receiver) = broadcast(128);
 
         let (task, source) = Self::create(num_storage_nodes, config, changes, change_sender).await;
-        run_builder_source(url, change_receiver, source);
+        run_builder_source_0_1(url, change_receiver, source);
         Box::new(task)
     }
 }

crates/testing/src/block_builder/simple.rs

@@ -24,14 +24,20 @@ use hotshot::{
     traits::BlockPayload,
     types::{Event, EventType, SignatureKey},
 };
-use hotshot_builder_api::v0_1::{
-    block_info::{AvailableBlockData, AvailableBlockHeaderInput, AvailableBlockInfo},
-    builder::{BuildError, Error, Options},
-    data_source::BuilderDataSource,
+use hotshot_builder_api::{
+    v0_1,
+    v0_1::{
+        block_info::{AvailableBlockData, AvailableBlockHeaderInput, AvailableBlockInfo},
+        builder::{BuildError, Error, Options},
+    },
+    v0_3,
 };
 use hotshot_types::{
+    bundle::Bundle,
+    constants::{LEGACY_BUILDER_MODULE, MARKETPLACE_BUILDER_MODULE},
     traits::{
-        block_contents::BlockHeader, node_implementation::NodeType,
+        block_contents::{BlockHeader, BuilderFee},
+        node_implementation::NodeType,
         signature_key::BuilderSignatureKey,
     },
     utils::BuilderCommitment,
@@ -126,7 +132,80 @@ impl<TYPES: NodeType> ReadState for SimpleBuilderSource<TYPES> {
 }
 
 #[async_trait]
-impl<TYPES: NodeType> BuilderDataSource<TYPES> for SimpleBuilderSource<TYPES>
+impl<TYPES: NodeType> v0_3::data_source::BuilderDataSource<TYPES> for SimpleBuilderSource<TYPES>
+where
+    <TYPES as NodeType>::InstanceState: Default,
+{
+    /// To get the list of available blocks
+    async fn bundle(
+        &self,
+        _parent_view: u64,
+        _parent_hash: &VidCommitment,
+        _view_number: u64,
+    ) -> Result<Bundle<TYPES>, BuildError> {
+        let transactions = self
+            .transactions
+            .read(|txns| {
+                Box::pin(async {
+                    txns.values()
+                        .filter(|txn| {
+                            // We want transactions that are either unclaimed, or claimed long ago
+                            // and thus probably not included, or they would've been decided on
+                            // already and removed from the queue
+                            txn.claimed
+                                .map(|claim_time| claim_time.elapsed() > Duration::from_secs(30))
+                                .unwrap_or(true)
+                        })
+                        .cloned()
+                        .map(|txn| txn.transaction)
+                        .collect::<Vec<TYPES::Transaction>>()
+                })
+            })
+            .await;
+
+        let fee_amount = 1;
+        let sequencing_fee: BuilderFee<TYPES> = BuilderFee {
+            fee_amount,
+            fee_account: self.pub_key.clone(),
+            fee_signature: TYPES::BuilderSignatureKey::sign_sequencing_fee_marketplace(
+                &self.priv_key.clone(),
+                fee_amount,
+            )
+            .expect("Failed to sign fee!"),
+        };
+
+        let signature =
+            TYPES::BuilderSignatureKey::sign_bundle::<TYPES>(&self.priv_key, &transactions)
+                .unwrap();
+
+        {
+            // claim transactions
+            let mut transactions_lock = self.transactions.write().await;
+            let transaction_hashes = transactions.iter().map(|txn| txn.commit());
+            let time = Instant::now();
+
+            for hash in transaction_hashes {
+                if let Some(txn) = transactions_lock.get_mut(&hash) {
+                    txn.claimed = Some(time);
+                }
+            }
+        }
+
+        Ok(Bundle {
+            transactions,
+            signature,
+            sequencing_fee,
+        })
+    }
+
+    /// To get the builder's address
+    async fn builder_address(&self) -> Result<TYPES::BuilderSignatureKey, BuildError> {
+        Ok(self.pub_key.clone())
+    }
+}
+
+#[async_trait]
+impl<TYPES: NodeType> v0_1::data_source::BuilderDataSource<TYPES> for SimpleBuilderSource<TYPES>
 where
     <TYPES as NodeType>::InstanceState: Default,
 {
@@ -243,14 +322,23 @@ impl<TYPES: NodeType> SimpleBuilderSource<TYPES> {
     where
         <TYPES as NodeType>::InstanceState: Default,
     {
-        let builder_api = hotshot_builder_api::v0_1::builder::define_api::<
+        let builder_api_0_1 = hotshot_builder_api::v0_1::builder::define_api::<
             SimpleBuilderSource<TYPES>,
             TYPES,
         >(&Options::default())
         .expect("Failed to construct the builder API");
+
+        let builder_api_0_3 = hotshot_builder_api::v0_3::builder::define_api::<
+            SimpleBuilderSource<TYPES>,
+            TYPES,
+        >(&Options::default())
+        .expect("Failed to construct the builder API");
+
         let mut app: App<SimpleBuilderSource<TYPES>, Error> = App::with_state(self);
-        app.register_module::<Error, _>("block_info", builder_api)
-            .expect("Failed to register the builder API");
+        app.register_module::<Error, _>(LEGACY_BUILDER_MODULE, builder_api_0_1)
+            .expect("Failed to register builder API 0.1")
+            .register_module::<Error, _>(MARKETPLACE_BUILDER_MODULE, builder_api_0_3)
+            .expect("Failed to register builder API 0.3");
 
         async_spawn(app.serve(url, hotshot_builder_api::v0_1::Version::instance()));
     }