Tb/weekly update plus

sequencer/src/lib.rs

@@ -482,7 +482,11 @@ pub async fn init_node<P: PersistenceOptions, V: Versions>(
         genesis_state.prefund_account(address, amount);
     }
 
-    let l1_client = l1_params.options.connect(l1_params.url).await?;
+    let l1_client = l1_params
+        .options
+        .with_metrics(metrics)
+        .connect(l1_params.url)
+        .await?;
     l1_client.spawn_tasks().await;
     let l1_genesis = match genesis.l1_finalized {
         L1Finalized::Block(b) => b,

sequencer/src/persistence.rs

@@ -9,7 +9,9 @@
 //! persistence which is _required_ to run a node.
 
 use async_trait::async_trait;
-use espresso_types::v0_99::ChainConfig;
+use committable::Committable;
+use espresso_types::{v0_99::ChainConfig, Leaf, Leaf2};
+use hotshot_types::{consensus::CommitmentMap, data::QuorumProposal};
 
 pub mod fs;
 pub mod no_storage;
@@ -20,6 +22,42 @@ pub trait ChainConfigPersistence: Sized + Send + Sync {
     async fn insert_chain_config(&mut self, chain_config: ChainConfig) -> anyhow::Result<()>;
 }
 
+fn downgrade_leaf(leaf2: Leaf2) -> Leaf {
+    if leaf2.drb_seed != [0; 96] && leaf2.drb_result != [0; 32] {
+        tracing::error!("Losing DRB information!");
+    }
+    let quorum_proposal = QuorumProposal {
+        block_header: leaf2.block_header().clone(),
+        view_number: leaf2.view_number(),
+        justify_qc: leaf2.justify_qc().to_qc(),
+        upgrade_certificate: leaf2.upgrade_certificate(),
+        proposal_certificate: None,
+    };
+    let mut leaf = Leaf::from_quorum_proposal(&quorum_proposal);
+    if let Some(payload) = leaf2.block_payload() {
+        leaf.fill_block_payload_unchecked(payload);
+    }
+    leaf
+}
+
+fn upgrade_commitment_map(map: CommitmentMap<Leaf>) -> CommitmentMap<Leaf2> {
+    map.into_values()
+        .map(|leaf| {
+            let leaf2: Leaf2 = leaf.into();
+            (leaf2.commit(), leaf2)
+        })
+        .collect()
+}
+
+fn downgrade_commitment_map(map: CommitmentMap<Leaf2>) -> CommitmentMap<Leaf> {
+    map.into_values()
+        .map(|leaf2| {
+            let leaf = downgrade_leaf(leaf2);
+            (<Leaf as Committable>::commit(&leaf), leaf)
+        })
+        .collect()
+}
+
 #[cfg(any(test, feature = "testing"))]
 mod testing {
 

sequencer/src/persistence/fs.rs

@@ -10,8 +10,8 @@ use hotshot_types::{
     consensus::CommitmentMap,
     data::{DaProposal, QuorumProposal, QuorumProposal2, VidDisperseShare},
     event::{Event, EventType, HotShotAction, LeafInfo},
-    message::Proposal,
-    simple_certificate::{QuorumCertificate2, UpgradeCertificate},
+    message::{convert_proposal, Proposal},
+    simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate},
     traits::{block_contents::BlockPayload, node_implementation::ConsensusTime},
     utils::View,
     vid::VidSchemeType,
@@ -28,6 +28,8 @@ use std::{
 
 use crate::ViewNumber;
 
+use super::{downgrade_commitment_map, downgrade_leaf, upgrade_commitment_map};
+
 /// Options for file system backed persistence.
 #[derive(Parser, Clone, Debug)]
 pub struct Options {
@@ -235,7 +237,7 @@ impl Inner {
             let bytes =
                 fs::read(&path).context(format!("reading decided leaf {}", path.display()))?;
             let (mut leaf, qc) =
-                bincode::deserialize::<(Leaf2, QuorumCertificate2<SeqTypes>)>(&bytes)
+                bincode::deserialize::<(Leaf, QuorumCertificate<SeqTypes>)>(&bytes)
                     .context(format!("parsing decided leaf {}", path.display()))?;
 
             // Include the VID share if available.
@@ -258,7 +260,7 @@ impl Inner {
             }
 
             let info = LeafInfo {
-                leaf,
+                leaf: leaf.into(),
                 vid_share,
 
                 // Note: the following fields are not used in Decide event processing, and should be
@@ -286,7 +288,7 @@ impl Inner {
                 .handle_event(&Event {
                     view_number: ViewNumber::new(view),
                     event: EventType::Decide {
-                        qc: Arc::new(qc),
+                        qc: Arc::new(qc.to_qc2()),
                         leaf_chain: Arc::new(vec![leaf]),
                         block_size: None,
                     },
@@ -346,14 +348,18 @@ impl Inner {
                 let bytes =
                     fs::read(&file).context(format!("reading decided leaf {}", file.display()))?;
                 let (leaf, qc) =
-                    bincode::deserialize::<(Leaf2, QuorumCertificate2<SeqTypes>)>(&bytes)
+                    bincode::deserialize::<(Leaf, QuorumCertificate<SeqTypes>)>(&bytes)
                         .context(format!("parsing decided leaf {}", file.display()))?;
                 if let Some((anchor_leaf, _)) = &anchor {
                     if leaf.view_number() > anchor_leaf.view_number() {
-                        anchor = Some((leaf, qc));
+                        let leaf2 = leaf.into();
+                        let qc2 = qc.to_qc2();
+                        anchor = Some((leaf2, qc2));
                     }
                 } else {
-                    anchor = Some((leaf, qc));
+                    let leaf2 = leaf.into();
+                    let qc2 = qc.to_qc2();
+                    anchor = Some((leaf2, qc2));
                 }
             }
 
@@ -447,7 +453,7 @@ impl SequencerPersistence for Persistence {
             fs::remove_file(&legacy_path).context("removing legacy anchor leaf file")?;
         }
 
-        for (info, qc) in leaf_chain {
+        for (info, qc2) in leaf_chain {
             let view = info.leaf.view_number().u64();
             let file_path = path.join(view.to_string()).with_extension("txt");
             inner.replace(
@@ -459,7 +465,9 @@ impl SequencerPersistence for Persistence {
                     Ok(false)
                 },
                 |mut file| {
-                    let bytes = bincode::serialize(&(&info.leaf, qc))?;
+                    let leaf = downgrade_leaf(info.leaf.clone());
+                    let qc = qc2.to_qc();
+                    let bytes = bincode::serialize(&(&leaf, qc))?;
                     file.write_all(&bytes)?;
                     Ok(())
                 },
@@ -500,7 +508,9 @@ impl SequencerPersistence for Persistence {
             return Ok(None);
         }
         let bytes = fs::read(&path).context("read")?;
-        Ok(Some(bincode::deserialize(&bytes).context("deserialize")?))
+        let value: (CommitmentMap<Leaf>, _) =
+            bincode::deserialize(&bytes).context("deserialize")?;
+        Ok(Some((upgrade_commitment_map(value.0), value.1)))
     }
 
     async fn load_da_proposal(
@@ -601,6 +611,8 @@ impl SequencerPersistence for Persistence {
         leaves: CommitmentMap<Leaf2>,
         state: BTreeMap<ViewNumber, View<SeqTypes>>,
     ) -> anyhow::Result<()> {
+        let leaves = downgrade_commitment_map(leaves);
+
         if !self.store_undecided_state {
             return Ok(());
         }
@@ -625,6 +637,8 @@ impl SequencerPersistence for Persistence {
         &self,
         proposal: &Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
     ) -> anyhow::Result<()> {
+        let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
+            convert_proposal(proposal.clone());
         let mut inner = self.inner.write().await;
         let view_number = proposal.data.view_number().u64();
         let dir_path = inner.quorum_proposals_dir_path();
@@ -689,11 +703,12 @@ impl SequencerPersistence for Persistence {
                 let proposal_bytes = fs::read(file)?;
 
                 // Then, deserialize.
-                let proposal: Proposal<SeqTypes, QuorumProposal2<SeqTypes>> =
+                let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
                     bincode::deserialize(&proposal_bytes)?;
+                let proposal2 = convert_proposal(proposal);
 
                 // Push to the map and we're done.
-                map.insert(view_number, proposal);
+                map.insert(view_number, proposal2);
             }
         }
 
@@ -708,8 +723,9 @@ impl SequencerPersistence for Persistence {
         let dir_path = inner.quorum_proposals_dir_path();
         let file_path = dir_path.join(view.to_string()).with_extension("txt");
         let bytes = fs::read(file_path)?;
-        let proposal = bincode::deserialize(&bytes)?;
-        Ok(proposal)
+        let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> = bincode::deserialize(&bytes)?;
+        let proposal2 = convert_proposal(proposal);
+        Ok(proposal2)
     }
 
     async fn load_upgrade_certificate(
@@ -758,7 +774,6 @@ impl SequencerPersistence for Persistence {
             Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
         ) -> Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
     ) -> anyhow::Result<()> {
-        // TODO:
         Ok(())
     }
 }

sequencer/src/persistence/sql.rs

@@ -21,8 +21,8 @@ use hotshot_types::{
     consensus::CommitmentMap,
     data::{DaProposal, QuorumProposal, QuorumProposal2, VidDisperseShare},
     event::{Event, EventType, HotShotAction, LeafInfo},
-    message::Proposal,
-    simple_certificate::{QuorumCertificate2, UpgradeCertificate},
+    message::{convert_proposal, Proposal},
+    simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate},
     traits::{node_implementation::ConsensusTime, BlockPayload},
     utils::View,
     vid::VidSchemeType,
@@ -36,6 +36,8 @@ use std::{collections::BTreeMap, time::Duration};
 
 use crate::{catchup::SqlStateCatchup, SeqTypes, ViewNumber};
 
+use super::{downgrade_commitment_map, downgrade_leaf, upgrade_commitment_map};
+
 /// Options for Postgres-backed persistence.
 #[derive(Parser, Clone, Derivative, Default)]
 #[derivative(Debug)]
@@ -315,9 +317,9 @@ impl Persistence {
             if hash.is_none() {
                 let view: i64 = row.try_get("view")?;
                 let data: Vec<u8> = row.try_get("data")?;
-                let proposal: Proposal<SeqTypes, QuorumProposal2<SeqTypes>> =
+                let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
                     bincode::deserialize(&data)?;
-                let leaf = Leaf2::from_quorum_proposal(&proposal.data);
+                let leaf = Leaf::from_quorum_proposal(&proposal.data);
                 let leaf_hash = Committable::commit(&leaf);
                 tracing::info!(view, %leaf_hash, "populating quorum proposal leaf hash");
                 updates.push((view, leaf_hash.to_string()));
@@ -376,14 +378,16 @@ impl SequencerPersistence for Persistence {
     ) -> anyhow::Result<()> {
         let values = leaf_chain
             .into_iter()
-            .map(|(info, qc)| {
+            .map(|(info, qc2)| {
                 // The leaf may come with a large payload attached. We don't care about this payload
                 // because we already store it separately, as part of the DA proposal. Storing it
                 // here contributes to load on the DB for no reason, so we remove it before
                 // serializing the leaf.
-                let mut leaf = info.leaf.clone();
+                let mut leaf = downgrade_leaf(info.leaf.clone());
                 leaf.unfill_block_payload();
 
+                let qc = qc2.to_qc();
+
                 let view = qc.view_number.u64() as i64;
                 let leaf_bytes = bincode::serialize(&leaf)?;
                 let qc_bytes = bincode::serialize(&qc)?;
@@ -440,12 +444,14 @@ impl SequencerPersistence for Persistence {
         };
 
         let leaf_bytes: Vec<u8> = row.get("leaf");
-        let leaf = bincode::deserialize(&leaf_bytes)?;
+        let leaf: Leaf = bincode::deserialize(&leaf_bytes)?;
+        let leaf2: Leaf2 = leaf.into();
 
         let qc_bytes: Vec<u8> = row.get("qc");
-        let qc = bincode::deserialize(&qc_bytes)?;
+        let qc: QuorumCertificate<SeqTypes> = bincode::deserialize(&qc_bytes)?;
+        let qc2 = qc.to_qc2();
 
-        Ok(Some((leaf, qc)))
+        Ok(Some((leaf2, qc2)))
     }
 
     async fn load_anchor_view(&self) -> anyhow::Result<ViewNumber> {
@@ -470,12 +476,13 @@ impl SequencerPersistence for Persistence {
         };
 
         let leaves_bytes: Vec<u8> = row.get("leaves");
-        let leaves = bincode::deserialize(&leaves_bytes)?;
+        let leaves: CommitmentMap<Leaf> = bincode::deserialize(&leaves_bytes)?;
+        let leaves2 = upgrade_commitment_map(leaves);
 
         let state_bytes: Vec<u8> = row.get("state");
         let state = bincode::deserialize(&state_bytes)?;
 
-        Ok(Some((leaves, state)))
+        Ok(Some((leaves2, state)))
     }
 
     async fn load_da_proposal(
@@ -536,9 +543,9 @@ impl SequencerPersistence for Persistence {
                     let view: i64 = row.get("view");
                     let view_number: ViewNumber = ViewNumber::new(view.try_into()?);
                     let bytes: Vec<u8> = row.get("data");
-                    let proposal: Proposal<SeqTypes, QuorumProposal2<SeqTypes>> =
+                    let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
                         bincode::deserialize(&bytes)?;
-                    Ok((view_number, proposal))
+                    Ok((view_number, convert_proposal(proposal)))
                 })
                 .collect::<anyhow::Result<Vec<_>>>()?,
         ))
@@ -554,7 +561,8 @@ impl SequencerPersistence for Persistence {
                 .bind(view.u64() as i64)
                 .fetch_one(tx.as_mut())
                 .await?;
-        let proposal = bincode::deserialize(&data)?;
+        let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> = bincode::deserialize(&data)?;
+        let proposal = convert_proposal(proposal);
         Ok(proposal)
     }
 
@@ -613,6 +621,8 @@ impl SequencerPersistence for Persistence {
         leaves: CommitmentMap<Leaf2>,
         state: BTreeMap<ViewNumber, View<SeqTypes>>,
     ) -> anyhow::Result<()> {
+        let leaves = downgrade_commitment_map(leaves);
+
         if !self.store_undecided_state {
             return Ok(());
         }
@@ -634,9 +644,11 @@ impl SequencerPersistence for Persistence {
         &self,
         proposal: &Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
     ) -> anyhow::Result<()> {
+        let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
+            convert_proposal(proposal.clone());
         let view_number = proposal.data.view_number().u64();
         let proposal_bytes = bincode::serialize(&proposal).context("serializing proposal")?;
-        let leaf_hash = Committable::commit(&Leaf2::from_quorum_proposal(&proposal.data));
+        let leaf_hash = Committable::commit(&Leaf::from_quorum_proposal(&proposal.data));
         let mut tx = self.db.write().await?;
         tx.upsert(
             "quorum_proposals",
@@ -694,7 +706,6 @@ impl SequencerPersistence for Persistence {
             Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
         ) -> Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
     ) -> anyhow::Result<()> {
-        // TODO:
         Ok(())
     }
 }