chore: rewrite block payload

builder/src/lib.rs

@@ -667,7 +667,7 @@ mod test {
         vid_commitment, BlockHeader, BlockPayload, EncodeBytes, GENESIS_VID_NUM_STORAGE_NODES,
     };
     use hotshot_types::utils::BuilderCommitment;
-    use sequencer::block::payload::Payload;
+    use sequencer::block::Payload;
     use sequencer::persistence::no_storage::{self, NoStorage};
     use sequencer::persistence::sql;
     use sequencer::{empty_builder_commitment, Header};

data/header.json

@@ -1,9 +1,9 @@
 {
   "block_merkle_tree_root": "MERKLE_COMM~AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAQA",
-  "builder_commitment": "BUILDER_COMMITMENT~PfISPEAHYbCXqJ08RqlK74d9aSrtrZkKHdnoX7crymG5",
+  "builder_commitment": "BUILDER_COMMITMENT~sb1x_a6aU3coBtMC2XowRwxDb3SJveIZxW-_Q9CSrkXd",
   "builder_signature": {
-    "r": "0xca39647d5e159ebf62e0efd163ddb2b7946f437948e6c31accdb5360a9f2da17",
-    "s": "0x610e51dced89ede20e3dcc5e9e26fff580738e785371df53389f2f84dadbdb87",
+    "r": "0x1596a279e9e0060b5d30577e56820f89a6bd0b5ab91efb70dc63f08b27e63cd0",
+    "s": "0x731ed939db42204098c6ffeb13812679576534e5cb3eeec335586402a28d99b5",
     "v": 28
   },
   "chain_config": {
@@ -29,9 +29,7 @@
     "timestamp": "0x456"
   },
   "l1_head": 124,
-  "ns_table": {
-    "bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
-  },
-  "payload_commitment": "HASH~AazstQer_ho1SqgGT0r10_Gs0BnjfbPBHJdSO3HHbp29",
+  "ns_table": "AQAAAO7/wAAAAAAACAQAAA==",
+  "payload_commitment": "HASH~Ojg3e_G9UzTwOygEDDcbvgakaVCqMByIiFrlkLXT72qf",
   "timestamp": 789
-}
+}

data/messages.json

@@ -8,7 +8,7 @@
             "data": {
               "block_header": {
                 "block_merkle_tree_root": "MERKLE_COMM~AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAQA",
-                "builder_commitment": "BUILDER_COMMITMENT~tEvs0rxqOiMCvfe2R0omNNaphSlUiEDrb2q0IZpRcgA_",
+                "builder_commitment": "BUILDER_COMMITMENT~_FlsTMQYXvjU_M1TpJFyrGe9BGPU9Di-qALdKJb2hegZ",
                 "builder_signature": null,
                 "chain_config": {
                   "chain_config": {
@@ -29,9 +29,7 @@
                 "height": 0,
                 "l1_finalized": null,
                 "l1_head": 0,
-                "ns_table": {
-                  "bytes": "AAAAAA=="
-                },
+                "ns_table": "AAAAAA==",
                 "payload_commitment": "HASH~AazstQer_ho1SqgGT0r10_Gs0BnjfbPBHJdSO3HHbp29",
                 "timestamp": 0
               },
@@ -319,9 +317,7 @@
                 2,
                 3
               ],
-              "metadata": {
-                "bytes": "AQAAAAEAAAALAAAA"
-              },
+              "metadata": "AQAAAAEAAAAAAAAACwAAAA==",
               "view_number": 0
             },
             "signature": "BLS_SIG~g3CUcLMD7fnDsBhItKvSqXLwEqdWfvusSrgpL1GBAxf-SWFW0t32Agt2jrOiempjjpI7dBwYGgXv-0mvI4sGEEE"

data/ns_table.json

@@ -1,3 +1 @@
-{
-    "bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
-}
+"AQAAAO7/wAAAAAAACAQAAA=="

sequencer/Cargo.toml

@@ -11,7 +11,9 @@ libp2p = []
 
 [dev-dependencies]
 espresso-macros = { git = "https://github.com/EspressoSystems/espresso-macros.git", tag = "0.1.0" }
+fluent-asserter = "0.1.9"
 hotshot-query-service = { workspace = true, features = ["testing"] }
+hotshot-testing = { workspace = true }
 pretty_assertions = { workspace = true }
 rand = "0.8.5"
 reqwest = { workspace = true }
@@ -54,6 +56,7 @@ es-version = { workspace = true }
 ethers = { workspace = true }
 ethers-contract-derive = "2.0.10"
 futures = { workspace = true }
+paste = "1.0"
 
 hotshot = { workspace = true }
 hotshot-events-service = { workspace = true }

sequencer/src/api.rs

@@ -620,16 +620,15 @@ mod api_tests {
     use crate::{
         persistence::no_storage,
         testing::{wait_for_decide_on_handle, TestConfig},
-        Header,
+        Header, NamespaceId,
     };
     use async_compatibility_layer::logging::{setup_backtrace, setup_logging};
     use committable::Committable;
     use data_source::testing::TestableSequencerDataSource;
     use endpoints::NamespaceProofQueryData;
     use es_version::SequencerVersion;
     use futures::stream::StreamExt;
-    use hotshot_query_service::availability::LeafQueryData;
-    use hotshot_types::vid::vid_scheme;
+    use hotshot_query_service::availability::{LeafQueryData, VidCommonQueryData};
     use portpicker::pick_unused_port;
     use surf_disco::Client;
     use test_helpers::{
@@ -661,8 +660,9 @@ mod api_tests {
         setup_logging();
         setup_backtrace();
 
-        let vid = vid_scheme(5);
-        let txn = Transaction::new(Default::default(), vec![1, 2, 3, 4]);
+        // Arbitrary transaction, arbitrary namespace ID
+        let ns_id = NamespaceId::from(42);
+        let txn = Transaction::new(ns_id, vec![1, 2, 3, 4]);
 
         // Start query service.
         let port = pick_unused_port().expect("No ports free");
@@ -711,14 +711,31 @@ mod api_tests {
                 .await
                 .unwrap();
             let ns_query_res: NamespaceProofQueryData = client
-                .get(&format!("availability/block/{block_num}/namespace/0"))
+                .get(&format!("availability/block/{block_num}/namespace/{ns_id}"))
                 .send()
                 .await
                 .unwrap();
-            ns_query_res
-                .proof
-                .verify(&vid, &header.payload_commitment, &header.ns_table)
-                .unwrap();
+
+            // Verify namespace proof if present
+            if let Some(ns_proof) = ns_query_res.proof {
+                let vid_common: VidCommonQueryData<SeqTypes> = client
+                    .get(&format!("availability/vid/common/{block_num}"))
+                    .send()
+                    .await
+                    .unwrap();
+
+                ns_proof
+                    .verify(
+                        &header.ns_table,
+                        &header.payload_commitment,
+                        vid_common.common(),
+                    )
+                    .unwrap();
+            } else {
+                // Namespace proof should be present if ns_id exists in ns_table
+                assert!(header.ns_table.find_ns_id(&ns_id).is_none());
+                assert!(ns_query_res.transactions.is_empty());
+            }
 
             found_empty_block = found_empty_block || ns_query_res.transactions.is_empty();
 

sequencer/src/api/endpoints.rs

@@ -14,10 +14,7 @@ use super::{
     StorageState,
 };
 use crate::{
-    block::payload::{parse_ns_payload, NamespaceProof},
-    network,
-    persistence::SequencerPersistence,
-    NamespaceId, SeqTypes, Transaction,
+    block::NsProof, network, persistence::SequencerPersistence, NamespaceId, SeqTypes, Transaction,
 };
 use anyhow::Result;
 use async_std::sync::{Arc, RwLock};
@@ -45,7 +42,7 @@ use vbs::version::StaticVersionType;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct NamespaceProofQueryData {
-    pub proof: NamespaceProof,
+    pub proof: Option<NsProof>,
     pub transactions: Vec<Transaction>,
 }
 
@@ -99,32 +96,25 @@ where
                 }
             )?;
 
-            let proof = block
-                .payload()
-                .namespace_with_proof(
-                    block.payload().get_ns_table(),
-                    ns_id,
-                    common.common().clone(),
-                )
-                .context(CustomSnafu {
-                    message: format!("failed to make proof for namespace {ns_id}"),
-                    status: StatusCode::NotFound,
-                })?;
-
-            let transactions = if let NamespaceProof::Existence {
-                ref ns_payload_flat,
-                ..
-            } = proof
-            {
-                parse_ns_payload(ns_payload_flat, ns_id)
+            if let Some(ns_index) = block.payload().ns_table().find_ns_id(&ns_id) {
+                let proof = NsProof::new(block.payload(), &ns_index, common.common()).context(
+                    CustomSnafu {
+                        message: format!("failed to make proof for namespace {ns_id}"),
+                        status: StatusCode::InternalServerError,
+                    },
+                )?;
+
+                Ok(NamespaceProofQueryData {
+                    transactions: proof.export_all_txs(&ns_id),
+                    proof: Some(proof),
+                })
             } else {
-                Vec::new()
-            };
-
-            Ok(NamespaceProofQueryData {
-                transactions,
-                proof,
-            })
+                // ns_id not found in ns_table
+                Ok(NamespaceProofQueryData {
+                    proof: None,
+                    transactions: Vec::new(),
+                })
+            }
         }
         .boxed()
     })?;
@@ -178,16 +168,6 @@ where
                 .body_auto::<Transaction, Ver>(Ver::instance())
                 .map_err(Error::from_request_error)?;
 
-            // Transactions with namespaces that do not fit in the u32
-            // cannot be included in the block.
-            // TODO: This issue will be addressed in the next release.
-            if tx.namespace() > NamespaceId::from(u32::MAX as u64) {
-                return Err(Error::Custom {
-                    message: "Transaction namespace > u32::MAX".to_string(),
-                    status: StatusCode::BadRequest,
-                });
-            }
-
             let hash = tx.commit();
             state
                 .submit(tx)

sequencer/src/bin/nasty-client.rs

@@ -31,14 +31,10 @@ use hotshot_query_service::{
     metrics::PrometheusMetrics,
     node::TimeWindowQueryData,
 };
-use hotshot_types::{
-    traits::metrics::{Counter, Gauge, Metrics as _},
-    vid::{vid_scheme, VidSchemeType},
-};
+use hotshot_types::traits::metrics::{Counter, Gauge, Metrics as _};
 use jf_merkle_tree::{
     ForgetableMerkleTreeScheme, MerkleCommitment, MerkleTreeScheme, UniversalMerkleTreeScheme,
 };
-use jf_vid::VidScheme;
 use rand::{seq::SliceRandom, RngCore};
 use sequencer::{
     api::endpoints::NamespaceProofQueryData,
@@ -940,11 +936,13 @@ impl ResourceManager<BlockQueryData<SeqTypes>> {
                     .context(format!("fetching header {block}"))
             })
             .await?;
-        if header.ns_table.is_empty() {
+        let num_namespaces = header.ns_table.iter().count();
+        if num_namespaces == 0 {
             tracing::info!("not fetching namespace because block {block} is empty");
             return Ok(());
         }
-        let ns = header.ns_table.get_table_entry(index).0;
+        let ns_index = header.ns_table.iter().nth(index % num_namespaces).unwrap();
+        let ns = header.ns_table.read_ns_id(&ns_index);
 
         let ns_proof: NamespaceProofQueryData = self
             .retry(info_span!("fetch namespace", %ns), || async {
@@ -966,13 +964,21 @@ impl ResourceManager<BlockQueryData<SeqTypes>> {
                     .context(format!("fetching VID common {block}"))
             })
             .await?;
-        let vid = vid_scheme(VidSchemeType::get_num_storage_nodes(vid_common.common()) as usize);
+        ensure!(
+            ns_proof.proof.is_some(),
+            format!("missing namespace proof for {block}:{ns}")
+        );
         ensure!(
             ns_proof
                 .proof
-                .verify(&vid, &header.payload_commitment, &header.ns_table)
+                .unwrap()
+                .verify(
+                    &header.ns_table,
+                    &header.payload_commitment,
+                    vid_common.common()
+                )
                 .is_some(),
-            format!("namespace proof for {block}:{ns} is invalid")
+            format!("failure to verify namespace proof for {block}:{ns}")
         );
 
         self.metrics.query_namespace_actions.add(1);