[CX-CLEANUP] - DA Integrated Storage

Cargo.toml

@@ -38,6 +38,7 @@ async-broadcast = "0.7.0"
 async-compatibility-layer = { git = "https://github.com/EspressoSystems/async-compatibility-layer.git", tag = "1.4.2", default-features = false, features = [
   "logging-utils",
 ] }
+anyhow = "1.0.81"
 task = { git = "https://github.com/EspressoSystems/HotShotTasks.git" }
 async-lock = "2.8"
 async-trait = "0.1.77"
@@ -59,13 +60,13 @@ futures = "0.3.30"
 # TODO generic-array should not be a direct dependency
 # https://github.com/EspressoSystems/HotShot/issues/1850
 generic-array = { version = "0.14.7", features = ["serde"] }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot-types", tag = "0.1.8" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot-types", tag = "0.1.10" }
 
 jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.2" }
 jf-plonk = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.2" }
 jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.2" }
 jf-utils = { git = "https://github.com/espressosystems/jellyfish", tag = "0.4.2" }
-hs-builder-api = { git = "https://github.com/EspressoSystems/hs-builder-api", tag = "0.1.3" }
+hs-builder-api = { git = "https://github.com/EspressoSystems/hs-builder-api", tag = "0.1.5" }
 lazy_static = "1.4.0"
 libp2p-identity = "0.2"
 libp2p-networking = { path = "./crates/libp2p-networking", version = "0.1.0", default-features = false }

crates/example-types/Cargo.toml

@@ -16,6 +16,8 @@ gpu-vid = [
 [dependencies]
 async-broadcast = { workspace = true }
 async-compatibility-layer = { workspace = true }
+async-trait = { workspace = true }
+anyhow = { workspace  = true }
 sha3 = "^0.10"
 bincode = { workspace = true }
 commit = { workspace = true }

crates/example-types/src/lib.rs

@@ -6,3 +6,6 @@ pub mod state_types;
 
 /// node types
 pub mod node_types;
+
+/// storage types for hotshot storage
+pub mod storage_types;

crates/example-types/src/node_types.rs

@@ -5,6 +5,7 @@ use hotshot::traits::{
 use crate::{
     block_types::{TestBlockHeader, TestBlockPayload, TestTransaction},
     state_types::{TestInstanceState, TestValidatedState},
+    storage_types::TestStorage,
 };
 
 use hotshot::traits::{
@@ -103,24 +104,29 @@ type StaticCombinedQuorumComm = CombinedNetworks<TestTypes>;
 impl NodeImplementation<TestTypes> for PushCdnImpl {
     type QuorumNetwork = StaticPushCdnQuorumComm;
     type CommitteeNetwork = StaticPushCdnDAComm;
+    type Storage = TestStorage<TestTypes>;
 }
 
 impl NodeImplementation<TestTypes> for Libp2pImpl {
     type QuorumNetwork = StaticLibp2pQuorumComm;
     type CommitteeNetwork = StaticLibp2pDAComm;
+    type Storage = TestStorage<TestTypes>;
 }
 
 impl NodeImplementation<TestTypes> for MemoryImpl {
     type QuorumNetwork = StaticMemoryQuorumComm;
     type CommitteeNetwork = StaticMemoryDAComm;
+    type Storage = TestStorage<TestTypes>;
 }
 
 impl NodeImplementation<TestTypes> for WebImpl {
     type QuorumNetwork = StaticWebQuorumComm;
     type CommitteeNetwork = StaticWebDAComm;
+    type Storage = TestStorage<TestTypes>;
 }
 
 impl NodeImplementation<TestTypes> for CombinedImpl {
     type QuorumNetwork = StaticCombinedQuorumComm;
     type CommitteeNetwork = StaticCombinedDAComm;
+    type Storage = TestStorage<TestTypes>;
 }

crates/example-types/src/storage_types.rs

@@ -0,0 +1,67 @@
+use anyhow::{bail, Result};
+use async_lock::RwLock;
+use async_trait::async_trait;
+use hotshot_types::{
+    data::{DAProposal, VidDisperse},
+    message::Proposal,
+    traits::{node_implementation::NodeType, storage::Storage},
+};
+use std::collections::HashMap;
+use std::sync::Arc;
+
+#[derive(Clone, Debug)]
+pub struct TestStorageState<TYPES: NodeType> {
+    vids: HashMap<TYPES::Time, Proposal<TYPES, VidDisperse<TYPES>>>,
+    das: HashMap<TYPES::Time, Proposal<TYPES, DAProposal<TYPES>>>,
+}
+
+impl<TYPES: NodeType> Default for TestStorageState<TYPES> {
+    fn default() -> Self {
+        Self {
+            vids: HashMap::new(),
+            das: HashMap::new(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct TestStorage<TYPES: NodeType> {
+    inner: Arc<RwLock<TestStorageState<TYPES>>>,
+
+    /// `should_return_err` is a testing utility to validate negative cases.
+    pub should_return_err: bool,
+}
+
+impl<TYPES: NodeType> Default for TestStorage<TYPES> {
+    fn default() -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(TestStorageState::default())),
+            should_return_err: false,
+        }
+    }
+}
+
+#[async_trait]
+impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
+    async fn append_vid(&self, proposal: &Proposal<TYPES, VidDisperse<TYPES>>) -> Result<()> {
+        if self.should_return_err {
+            bail!("Failed to append VID proposal to storage");
+        }
+        let mut inner = self.inner.write().await;
+        inner
+            .vids
+            .insert(proposal.data.view_number, proposal.clone());
+        Ok(())
+    }
+
+    async fn append_da(&self, proposal: &Proposal<TYPES, DAProposal<TYPES>>) -> Result<()> {
+        if self.should_return_err {
+            bail!("Failed to append VID proposal to storage");
+        }
+        let mut inner = self.inner.write().await;
+        inner
+            .das
+            .insert(proposal.data.view_number, proposal.clone());
+        Ok(())
+    }
+}

crates/examples/combined/types.rs

@@ -1,6 +1,6 @@
 use crate::infra::CombinedDARun;
 use hotshot::traits::implementations::CombinedNetworks;
-use hotshot_example_types::state_types::TestTypes;
+use hotshot_example_types::{state_types::TestTypes, storage_types::TestStorage};
 use hotshot_types::traits::node_implementation::NodeImplementation;
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
@@ -21,6 +21,7 @@ pub type ViewSyncNetwork = CombinedNetworks<TestTypes>;
 impl NodeImplementation<TestTypes> for NodeImpl {
     type QuorumNetwork = QuorumNetwork;
     type CommitteeNetwork = DANetwork;
+    type Storage = TestStorage<TestTypes>;
 }
 /// convenience type alias
 pub type ThisRun = CombinedDARun<TestTypes>;

crates/examples/infra/mod.rs

@@ -20,6 +20,7 @@ use hotshot::{
     types::{SignatureKey, SystemContextHandle},
     Memberships, Networks, SystemContext,
 };
+use hotshot_example_types::storage_types::TestStorage;
 use hotshot_example_types::{
     block_types::{TestBlockHeader, TestBlockPayload, TestTransaction},
     state_types::TestInstanceState,
@@ -429,7 +430,12 @@ pub trait RunDA<
     TYPES: NodeType<InstanceState = TestInstanceState>,
     DANET: ConnectedNetwork<Message<TYPES>, TYPES::SignatureKey>,
     QUORUMNET: ConnectedNetwork<Message<TYPES>, TYPES::SignatureKey>,
-    NODE: NodeImplementation<TYPES, QuorumNetwork = QUORUMNET, CommitteeNetwork = DANET>,
+    NODE: NodeImplementation<
+        TYPES,
+        QuorumNetwork = QUORUMNET,
+        CommitteeNetwork = DANET,
+        Storage = TestStorage<TYPES>,
+    >,
 > where
     <TYPES as NodeType>::ValidatedState: TestableState<TYPES>,
     <TYPES as NodeType>::BlockPayload: TestableBlock,
@@ -506,6 +512,7 @@ pub trait RunDA<
             networks_bundle,
             initializer,
             ConsensusMetricsValue::default(),
+            TestStorage::<TYPES>::default(),
         )
         .await
         .expect("Could not init hotshot")
@@ -705,6 +712,7 @@ impl<
             TYPES,
             QuorumNetwork = WebServerNetwork<TYPES>,
             CommitteeNetwork = WebServerNetwork<TYPES>,
+            Storage = TestStorage<TYPES>,
         >,
     > RunDA<TYPES, WebServerNetwork<TYPES>, WebServerNetwork<TYPES>, NODE> for WebServerDARun<TYPES>
 where
@@ -778,6 +786,7 @@ impl<
             TYPES,
             QuorumNetwork = PushCdnNetwork<TYPES>,
             CommitteeNetwork = PushCdnNetwork<TYPES>,
+            Storage = TestStorage<TYPES>,
         >,
     > RunDA<TYPES, PushCdnNetwork<TYPES>, PushCdnNetwork<TYPES>, NODE> for PushCdnDaRun<TYPES>
 where
@@ -860,6 +869,7 @@ impl<
             TYPES,
             QuorumNetwork = Libp2pNetwork<Message<TYPES>, TYPES::SignatureKey>,
             CommitteeNetwork = Libp2pNetwork<Message<TYPES>, TYPES::SignatureKey>,
+            Storage = TestStorage<TYPES>,
         >,
     >
     RunDA<
@@ -929,6 +939,7 @@ impl<
             TYPES,
             QuorumNetwork = CombinedNetworks<TYPES>,
             CommitteeNetwork = CombinedNetworks<TYPES>,
+            Storage = TestStorage<TYPES>,
         >,
     > RunDA<TYPES, CombinedNetworks<TYPES>, CombinedNetworks<TYPES>, NODE> for CombinedDARun<TYPES>
 where
@@ -1014,7 +1025,12 @@ pub async fn main_entry_point<
     >,
     DACHANNEL: ConnectedNetwork<Message<TYPES>, TYPES::SignatureKey>,
     QUORUMCHANNEL: ConnectedNetwork<Message<TYPES>, TYPES::SignatureKey>,
-    NODE: NodeImplementation<TYPES, QuorumNetwork = QUORUMCHANNEL, CommitteeNetwork = DACHANNEL>,
+    NODE: NodeImplementation<
+        TYPES,
+        QuorumNetwork = QUORUMCHANNEL,
+        CommitteeNetwork = DACHANNEL,
+        Storage = TestStorage<TYPES>,
+    >,
     RUNDA: RunDA<TYPES, DACHANNEL, QUORUMCHANNEL, NODE>,
 >(
     args: ValidatorArgs,

crates/examples/libp2p/types.rs

@@ -1,6 +1,6 @@
 use crate::infra::Libp2pDARun;
 use hotshot::traits::implementations::Libp2pNetwork;
-use hotshot_example_types::state_types::TestTypes;
+use hotshot_example_types::{state_types::TestTypes, storage_types::TestStorage};
 use hotshot_types::{
     message::Message,
     traits::node_implementation::{NodeImplementation, NodeType},
@@ -20,6 +20,7 @@ pub type QuorumNetwork = Libp2pNetwork<Message<TestTypes>, <TestTypes as NodeTyp
 impl NodeImplementation<TestTypes> for NodeImpl {
     type QuorumNetwork = QuorumNetwork;
     type CommitteeNetwork = DANetwork;
+    type Storage = TestStorage<TestTypes>;
 }
 /// convenience type alias
 pub type ThisRun = Libp2pDARun<TestTypes>;

crates/examples/push-cdn/types.rs

@@ -1,6 +1,6 @@
 use crate::infra::PushCdnDaRun;
 use hotshot::traits::implementations::PushCdnNetwork;
-use hotshot_example_types::state_types::TestTypes;
+use hotshot_example_types::{state_types::TestTypes, storage_types::TestStorage};
 use hotshot_types::traits::node_implementation::NodeImplementation;
 use serde::{Deserialize, Serialize};
 
@@ -20,6 +20,7 @@ pub type ViewSyncNetwork = PushCdnNetwork<TestTypes>;
 impl NodeImplementation<TestTypes> for NodeImpl {
     type CommitteeNetwork = DANetwork;
     type QuorumNetwork = QuorumNetwork;
+    type Storage = TestStorage<TestTypes>;
 }
 
 /// Convenience type alias