Merge pull request #1449 from EspressoSystems/jb/undecided-state

Store undecided consensus state and load it on restart
EspressoSystems · May 10, 2024 · 73f21f2 · 73f21f2
2 parents ddeeab7 + d736f16
commit 73f21f2
Show file tree

Hide file tree

Showing 8 changed files with 173 additions and 35 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -46,18 +46,18 @@ dotenvy = "0.15"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.51" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.53" }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.51" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.19" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.20" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.51" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.20" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.51" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.53" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.20" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.21" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.53" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.21" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.53" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.51" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.51" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.51" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.53" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.53" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.53" }
 
 # Push CDN imports
 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [

diff --git a/sequencer/api/migrations/V15__undecided_state.sql b/sequencer/api/migrations/V15__undecided_state.sql
@@ -0,0 +1,8 @@
+CREATE TABLE undecided_state (
+    -- The ID is always set to 0. Setting it explicitly allows us to enforce with every insert or
+    -- update that there is only a single entry in this table: the latest known state.
+    id INT PRIMARY KEY,
+
+    leaves BYTEA NOT NULL,
+    state  BYTEA NOT NULL
+);
diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs
@@ -145,10 +145,13 @@ impl<P: SequencerPersistence> Storage<SeqTypes> for Arc<RwLock<P>> {
 
     async fn update_undecided_state(
         &self,
-        _leaves: CommitmentMap<Leaf>,
-        _state: BTreeMap<ViewNumber, View<SeqTypes>>,
+        leaves: CommitmentMap<Leaf>,
+        state: BTreeMap<ViewNumber, View<SeqTypes>>,
     ) -> anyhow::Result<()> {
-        Ok(())
+        self.write()
+            .await
+            .update_undecided_state(leaves, state)
+            .await
     }
 }
 

diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs
@@ -19,13 +19,15 @@ use hotshot::{
     HotShotInitializer,
 };
 use hotshot_types::{
+    consensus::CommitmentMap,
     data::{DAProposal, VidDisperseShare},
     event::{HotShotAction, LeafInfo},
     message::Proposal,
     simple_certificate::QuorumCertificate,
     traits::node_implementation::ConsensusTime,
+    utils::View,
 };
-use std::cmp::max;
+use std::{cmp::max, collections::BTreeMap};
 
 pub mod fs;
 pub mod no_storage;
@@ -80,6 +82,11 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
     async fn load_anchor_leaf(&self)
         -> anyhow::Result<Option<(Leaf, QuorumCertificate<SeqTypes>)>>;
 
+    /// Load undecided state saved by consensus before we shut down.
+    async fn load_undecided_state(
+        &self,
+    ) -> anyhow::Result<Option<(CommitmentMap<Leaf>, BTreeMap<ViewNumber, View<SeqTypes>>)>>;
+
     async fn load_vid_share(
         &self,
         view: ViewNumber,
@@ -144,15 +151,28 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
             view += 1;
         }
 
-        tracing::info!(?leaf, ?view, ?high_qc, "loaded consensus state");
+        let (undecided_leaves, undecided_state) = self
+            .load_undecided_state()
+            .await
+            .context("loading undecided state")?
+            .unwrap_or_default();
+
+        tracing::info!(
+            ?leaf,
+            ?view,
+            ?high_qc,
+            ?undecided_leaves,
+            ?undecided_state,
+            "loaded consensus state"
+        );
         Ok(HotShotInitializer::from_reload(
             leaf,
             state,
             validated_state,
             view,
             high_qc,
-            Default::default(),
-            Default::default(),
+            undecided_leaves.into_values().collect(),
+            undecided_state,
         ))
     }
 
@@ -196,6 +216,11 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static {
         view: ViewNumber,
         action: HotShotAction,
     ) -> anyhow::Result<()>;
+    async fn update_undecided_state(
+        &mut self,
+        leaves: CommitmentMap<Leaf>,
+        state: BTreeMap<ViewNumber, View<SeqTypes>>,
+    ) -> anyhow::Result<()>;
 }
 
 #[cfg(test)]

diff --git a/sequencer/src/persistence/fs.rs b/sequencer/src/persistence/fs.rs
@@ -5,14 +5,17 @@ use async_trait::async_trait;
 use clap::Parser;
 
 use hotshot_types::{
+    consensus::CommitmentMap,
     data::{DAProposal, VidDisperseShare},
     event::HotShotAction,
     message::Proposal,
     simple_certificate::QuorumCertificate,
     traits::node_implementation::ConsensusTime,
+    utils::View,
     vote::HasViewNumber,
 };
 use std::{
+    collections::BTreeMap,
     fs::{self, File, OpenOptions},
     io::{Read, Seek, SeekFrom, Write},
     path::{Path, PathBuf},
@@ -70,6 +73,10 @@ impl Persistence {
         self.0.join("da")
     }
 
+    fn undecided_state_path(&self) -> PathBuf {
+        self.0.join("undecided_state")
+    }
+
     /// Overwrite a file if a condition is met.
     ///
     /// The file at `path`, if it exists, is opened in read mode and passed to `pred`. If `pred`
@@ -234,6 +241,17 @@ impl SequencerPersistence for Persistence {
         Ok(Some(bincode::deserialize(&bytes).context("deserialize")?))
     }
 
+    async fn load_undecided_state(
+        &self,
+    ) -> anyhow::Result<Option<(CommitmentMap<Leaf>, BTreeMap<ViewNumber, View<SeqTypes>>)>> {
+        let path = self.undecided_state_path();
+        if !path.is_file() {
+            return Ok(None);
+        }
+        let bytes = fs::read(&path).context("read")?;
+        Ok(Some(bincode::deserialize(&bytes).context("deserialize")?))
+    }
+
     async fn load_da_proposal(
         &self,
         view: ViewNumber,
@@ -349,6 +367,25 @@ impl SequencerPersistence for Persistence {
             },
         )
     }
+    async fn update_undecided_state(
+        &mut self,
+        leaves: CommitmentMap<Leaf>,
+        state: BTreeMap<ViewNumber, View<SeqTypes>>,
+    ) -> anyhow::Result<()> {
+        self.replace(
+            &self.undecided_state_path(),
+            |_| {
+                // Always overwrite the previous file.
+                Ok(true)
+            },
+            |mut file| {
+                let bytes =
+                    bincode::serialize(&(leaves, state)).context("serializing undecided state")?;
+                file.write_all(&bytes)?;
+                Ok(())
+            },
+        )
+    }
 }
 
 #[cfg(test)]

diff --git a/sequencer/src/persistence/no_storage.rs b/sequencer/src/persistence/no_storage.rs
@@ -5,11 +5,14 @@ use super::{NetworkConfig, PersistenceOptions, SequencerPersistence};
 use crate::{Leaf, SeqTypes, ViewNumber};
 use async_trait::async_trait;
 use hotshot_types::{
+    consensus::CommitmentMap,
     data::{DAProposal, VidDisperseShare},
     event::HotShotAction,
     message::Proposal,
     simple_certificate::QuorumCertificate,
+    utils::View,
 };
+use std::collections::BTreeMap;
 
 #[derive(Clone, Copy, Debug)]
 pub struct Options;
@@ -62,6 +65,12 @@ impl SequencerPersistence for NoStorage {
         Ok(None)
     }
 
+    async fn load_undecided_state(
+        &self,
+    ) -> anyhow::Result<Option<(CommitmentMap<Leaf>, BTreeMap<ViewNumber, View<SeqTypes>>)>> {
+        Ok(None)
+    }
+
     async fn load_da_proposal(
         &self,
         _view: ViewNumber,
@@ -95,4 +104,11 @@ impl SequencerPersistence for NoStorage {
     ) -> anyhow::Result<()> {
         Ok(())
     }
+    async fn update_undecided_state(
+        &mut self,
+        _leaves: CommitmentMap<Leaf>,
+        _state: BTreeMap<ViewNumber, View<SeqTypes>>,
+    ) -> anyhow::Result<()> {
+        Ok(())
+    }
 }