chore: rewrite block payload (#1499)

* new struct Payload2

* WIP new fns usize_to_bytes, max_from_byte_len with tests

* implement NamespaceBuider

* WIP begin implementing from_transactions

* dead end: const generics not stable in Rust https://stackoverflow.com/a/72467535

* finish impl for from_transactions, use macro_rules to generalize usize_to_bytes

* WIP friendly deserializers

* generalized friendly deserializer

* usize_from_bytes const generic param, add xxx_from_bytes functions

* impl namespace_with_proof and some helpers

* WIP test infra for namespace proofs

* tweak test

* Payload2:namespace_with_proof pass tests (yay)

* don't double count dupliate namespace ids

* tidy

* restore block.rs from main, new file block2.rs

* move mod tx_table to separate file payload_bytes.rs

* rename block2::Payload2 -> Payload

* set Payload::ns_iter() Item to NamespaceId

* move namespace iterator to a separate file

* rename payload2 -> ns_payload_builder

* visibility tweaks for ns_iter

* new fn verify_namespace_proof, temporary re-use of old parse_ns_payload, enforce maximum ns payload index in ns_iter, rename a few things

* move namespace_with_proof and test to ns_proof.rs, use new verify_namespace_proof in test

* move Payload::ns_iter, etc to ns_iter.rs

* rename ns_payload_builder -> ns_payload

* new mod tx_iter, a proper impl for parse_ns_payload

* WIP combined iterator for QueryablePayload

* move the combined iterator to iter.rs, delete the extra namespace iterator

* stub impl of QueryablePayload for Payload

* more stubs, tidy, new file tx_proof.rs

* fix bug in TxIter, fix test

* impl Payload::transaction with test

* move tests to new file test.rs

* tidy and comments

* NsProof do not store VID common data

* tidying and stub

* new fn tx_table_range with doc

* impl transaction_with_proof, still pretty messy tho

* WIP tx proof only for num_txs

* fix bug in iter, fix test

* test: verify transaction proofs

* major rework of ns_iter: new struct NsTable, NsIter::Item is now just usize

* newtype NsIndex

* TxIndex is now a serialized index, newtype NsPayload with awesome helper methods

* fix name _max_from_byte_len2 -> _max_from_byte_len

* xxx_from_bytes allow small byte lengths

* NsIndex in serialized form like TxIndex

* move tx_iter mod into ns_payload to enable private data in TxIndex

* rename module ns_iter -> ns_table

* tweak todo comments

* move ns_payload, ns_proof, tx_proof modules inside ns_table

* tidy

* put TxIndex in a new mod tx_iter, move NsPayload::read_tx_offset into tx_iter, new method NsPayload::read_tx_offset_pref

* add tx table range proof to TxProof

* NsPayload now a DST, add newtype NsPayloadOwned so that it's to NsPayload as Vec<T> is to [T]

* untested: TxProof::verify check tx table proof

* dumbest bug ever

* TxProof::verify now check tx payload proof (yay)

* tidy: new module ns_iter like tx_iter, new method NsTable::read_ns_offset_prev like NsPayload::read_tx_offset_prev

* new struct NsPayloadRange with helpers

* WIP tweak tx_payload_range[_relative]

* make NsPayloadRange a Range<usize>

* WIP prep for experiments with NsPayload

* add range field to NsPayload, it can no longer be a DST (boo)

* revert back to DST for NsPayload

* move tx_payload_range method from NsPayload to NsPayloadRange, add args to make it work

* move modules ns_proof, tx_proof from ns_table up to block

* move module ns_payload_range to its own file

* Index fields private

* move module ns_iter to its own file

* move module tx_iter to its own file

* newtype NumTxs

* manual serde impl for NumTxs

* NsPayloadRange::tx_payload_range arg type change usize -> NumTxs

* new struct TxTableEntries

* manual serde impl for TxTableEntries

* tidy ns_table

* move module num_txs into its own file

* move module tx_table_entries to its own file

* remove pub(crate) from NsPayloadRange newtype field

* add TODOs, ugh Rust is killing me

* TxIndex newtype from array to usize

* NsIndex newtype from array to usize

* move module num_txs up to block, experiment with access key design pattern

* move module ns_iter up to block

* move module tx_table_entries up to block

* move module tx_iter up to block

* move module ns_payload up to block

* move module ns_payload_range up to block

* move some NsTable impls into ns_table module

* NsTable member private

* move some impl Payload to block module

* move NsProof construction from Payload to NsProof

* move TxProof construction from Payload to TxProof

* move struct Payload to a new module payload

* visibility restrictions to payload module

* oops

* delete num_txs_as_bytes from payload_bytes

* delete num_txs_from_bytes from payload_bytes

* delete tx_offset_as_bytes from payload_bytes

* delete tx_offset_from_bytes from payload_bytes

* delete num_nss_as_bytes from payload_bytes

* delete num_nss_from_bytes from payload_bytes

* delete ns_offset_as_bytes from payload_bytes

* delete ns_offset_from_bytes from payload_bytes

* delete ns_offset_[as|from]_bytes from payload_bytes

* tweak: Payload::ns_payload re-use ns_payload_range

* move byte len constants to block module

* rename module payload_bytes -> uint_bytes

* tidy, minor refactor new function usize_fits

* replace NsTable::num_nss_with_duplicates -> in_bounds, reflect the change in NsIter, use it in TxProof; also remove NsPayload::find_ns_id and reorder arg list in TxProof::new

* check tx index in TxProof::new, new method NsPayload::in_bounds

* WIP new model for NsPayload[Range]

* WIP read_tx_offset

* new traits AsBytes, BytesReader, new test for TxProof2

* PoC TxTableEntries in the new model

* tidy, rename

* remove const generic param from AsPayloadBytes trait

* new structs NumTxs2, TxTableEntries2 using traits AsPayloadBytes

* add tx payload range to TxProof2

* error checking in TxProof::new

* TxProof::verify: add ns_table arg, remove ns_payload_range from proof, add error checking

* derive serde for types in TxProof2

* delete old type TxProof in favor of TxProof2

* NsProofExistence use NsPayloadOwned2 instead of NsPayloadOwned

* Iter use TxIter::new2 instead of new (progress toward switching from NsPayload to NsPayload2)

* move NamespacePayloadBuilder to module newtypes

* delete module ns_payload_range

* delete old modules

* newtype NsPayloadByteLen

* newtype NumTxsChecked

* move tx_table_entries_range_relative into TxTableEntriesRange::new

* move module tx_iter into newtypes

* impl AsPayloadBytes for TxIndex

* WIP test fails: AsPayloadBytes new param T

* fix test, but AsPayloadBytes trait is now unusable (boo)

* fix TxTableEntries deserialization

* delete unneeded stuff

* rename a bunch of types in module newtypes

* make AsPayloadBytes readable and rename it to FromPayloadBytes

* tidy and rename

* rename ns_payload[_range]2.rx -> without the 2

* tidy and renaming

* newtype PayloadByteLen

* tidy and docs

* tidy ns_table

* tidy payload

* fix macro bytes_serde_impl

* delete ns_iter.rs, move contents to ns_table.rs

* restrict visibility of magic constants to a single file (yay)

* tidy ns_payload

* replace NsPayloadRange::offset with block_payload_range, simplify NsPayloadBytesRange

* tidy tx_proof, rename some things

* tidy

* new method export_tx, in prep for reduced visibility

* fix use statements

* new module full_payload

* WIP new module namespace_payload

* move tx_proof, iter to namespace_payload; add helpers to avoid excessive visibility

* new helper Payload::ns_payload

* doc for bytes_serde_impl macro

* move ns_payload_traits module into newtypes

* rename module newtypes -> types

* fix build after merge main

* WIP swap out block for block2

* WIP fix test_namespace_query for block2

* WIP fix nasty_client except for NsIndex serialization issue

* fix nasty-client for new block2, appease clippy

* fix reference test for new ns table format

* fix test demo, tidy

* accounting for block byte length limit

* temporary hack to pass tests

* set forge-std git submodule to correct commit

* fix test_message_compat

* failing test for large namespace ids

* single-character bug fix (damn that feels good)

* fix doctest

* update reference tests (again)

* add test enforce_max_block_size

* delete old block module

* tidy TODOs, some general tidying

* NsTable::read_ns_id check index in bounds

* NsTable::ns_range check index in bounds

* use read_ns_id_unchecked in NsIter

* revert NsTable::ns_range to unchecked and restrict visibility

* revert NsTable::read_ns_id to unchecked

* re-arrange methods

* NsTable::as_bytes_slice restrict visibility

* delete NsTable::as_bytes_slice in favor of Encode trait

* delete Payload::as_byte_slice in favor of Encode trait

* tidy PayloadByteLen

* restrict visibility for NsIter

* restrict visibility of PayloadByteLen

* restrict visibility of NsPayload

* restrict visibility of NsPayloadRange

* restrict visibility of NsPayloadBuilder

* restrict visibility of TxIndex, TxIter

* rename module block2 -> block

* rename ns_payload_unchecked -> ns_payload

* remove obsolete todo

* revert #1504 ; this PR supports arbitrary 8-byte namespace IDs

* detailed specification rustdoc for Payload, NsTable

* detailed specification in rustdoc for namespace payload

* rename Payload::payload -> ns_payloads

* NsProof do not prove non-existence

* Payload::is_consistent return Result instead of bool, eliminate panic

* NsProof::new take NsIndex arg instead of NamespaceId

* NamespaceProofQueryData::proof is now optional

* fix: NsProof::ns_proof for empty payload should be None

* address #1499 (comment)

* NsTable  field for backwards compatibility

* set NS_ID_BYTE_LEN to 4 for backwards compatibility

* Payload::builder_commitment hack for backwards compatibility

* TODOs for NamespaceId, fix tests in block/test.rs

* restore data/ files from main branch

* remove obsolete comment

* fix doc for NsProof::new as per #1499 (comment)

* new method NsTable::read_ns_id_unchecked as per #1499 (comment)

* NamespaceId manual Deserialize impl enforce u32::MAX

* NamespaceId impl From<u32> as per #1499 (comment)

* doc: links to github issues in code comments

builder/src/lib.rs

@@ -669,7 +669,7 @@ mod test {
         vid_commitment, BlockHeader, BlockPayload, EncodeBytes, GENESIS_VID_NUM_STORAGE_NODES,
     };
     use hotshot_types::utils::BuilderCommitment;
-    use sequencer::block::payload::Payload;
+    use sequencer::block::Payload;
     use sequencer::persistence::no_storage::{self, NoStorage};
     use sequencer::persistence::sql;
     use sequencer::{empty_builder_commitment, Header};

sequencer/Cargo.toml

@@ -16,6 +16,7 @@ required-features = ["testing"]
 [dev-dependencies]
 escargot = "0.5.10"
 espresso-macros = { git = "https://github.com/EspressoSystems/espresso-macros.git", tag = "0.1.0" }
+fluent-asserter = "0.1.9"
 hotshot-query-service = { workspace = true, features = ["testing"] }
 hotshot-testing = { workspace = true }
 pretty_assertions = { workspace = true }
@@ -60,6 +61,7 @@ es-version = { workspace = true }
 ethers = { workspace = true }
 ethers-contract-derive = "2.0.10"
 futures = { workspace = true }
+paste = "1.0"
 
 hotshot = { workspace = true }
 hotshot-contract-adapter = { workspace = true }

sequencer/src/api.rs

@@ -687,7 +687,7 @@ mod api_tests {
     use crate::{
         persistence::no_storage,
         testing::{wait_for_decide_on_handle, TestConfig},
-        Header,
+        Header, NamespaceId,
     };
     use async_compatibility_layer::logging::{setup_backtrace, setup_logging};
     use committable::Committable;
@@ -696,8 +696,7 @@ mod api_tests {
     use es_version::SequencerVersion;
     use ethers::utils::Anvil;
     use futures::stream::StreamExt;
-    use hotshot_query_service::availability::LeafQueryData;
-    use hotshot_types::vid::vid_scheme;
+    use hotshot_query_service::availability::{LeafQueryData, VidCommonQueryData};
     use portpicker::pick_unused_port;
     use surf_disco::Client;
     use test_helpers::{
@@ -729,8 +728,9 @@ mod api_tests {
         setup_logging();
         setup_backtrace();
 
-        let vid = vid_scheme(5);
-        let txn = Transaction::new(Default::default(), vec![1, 2, 3, 4]);
+        // Arbitrary transaction, arbitrary namespace ID
+        let ns_id = NamespaceId::from(42);
+        let txn = Transaction::new(ns_id, vec![1, 2, 3, 4]);
 
         // Start query service.
         let port = pick_unused_port().expect("No ports free");
@@ -783,14 +783,31 @@ mod api_tests {
                 .await
                 .unwrap();
             let ns_query_res: NamespaceProofQueryData = client
-                .get(&format!("availability/block/{block_num}/namespace/0"))
+                .get(&format!("availability/block/{block_num}/namespace/{ns_id}"))
                 .send()
                 .await
                 .unwrap();
-            ns_query_res
-                .proof
-                .verify(&vid, &header.payload_commitment, &header.ns_table)
-                .unwrap();
+
+            // Verify namespace proof if present
+            if let Some(ns_proof) = ns_query_res.proof {
+                let vid_common: VidCommonQueryData<SeqTypes> = client
+                    .get(&format!("availability/vid/common/{block_num}"))
+                    .send()
+                    .await
+                    .unwrap();
+
+                ns_proof
+                    .verify(
+                        &header.ns_table,
+                        &header.payload_commitment,
+                        vid_common.common(),
+                    )
+                    .unwrap();
+            } else {
+                // Namespace proof should be present if ns_id exists in ns_table
+                assert!(header.ns_table.find_ns_id(&ns_id).is_none());
+                assert!(ns_query_res.transactions.is_empty());
+            }
 
             found_empty_block = found_empty_block || ns_query_res.transactions.is_empty();
 

sequencer/src/api/endpoints.rs

@@ -14,10 +14,7 @@ use super::{
     StorageState,
 };
 use crate::{
-    block::payload::{parse_ns_payload, NamespaceProof},
-    network,
-    persistence::SequencerPersistence,
-    NamespaceId, SeqTypes, Transaction,
+    block::NsProof, network, persistence::SequencerPersistence, NamespaceId, SeqTypes, Transaction,
 };
 use anyhow::Result;
 use async_std::sync::{Arc, RwLock};
@@ -45,7 +42,7 @@ use vbs::version::StaticVersionType;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct NamespaceProofQueryData {
-    pub proof: NamespaceProof,
+    pub proof: Option<NsProof>,
     pub transactions: Vec<Transaction>,
 }
 
@@ -74,8 +71,7 @@ where
     api.get("getnamespaceproof", move |req, state| {
         async move {
             let height: usize = req.integer_param("height")?;
-            let ns_id: u64 = req.integer_param("namespace")?;
-            let ns_id = NamespaceId::from(ns_id);
+            let ns_id = NamespaceId::from(req.integer_param::<_, u32>("namespace")?);
             let (block, common) = try_join!(
                 async move {
                     state
@@ -99,32 +95,25 @@ where
                 }
             )?;
 
-            let proof = block
-                .payload()
-                .namespace_with_proof(
-                    block.payload().get_ns_table(),
-                    ns_id,
-                    common.common().clone(),
-                )
-                .context(CustomSnafu {
-                    message: format!("failed to make proof for namespace {ns_id}"),
-                    status: StatusCode::NOT_FOUND,
-                })?;
-
-            let transactions = if let NamespaceProof::Existence {
-                ref ns_payload_flat,
-                ..
-            } = proof
-            {
-                parse_ns_payload(ns_payload_flat, ns_id)
+            if let Some(ns_index) = block.payload().ns_table().find_ns_id(&ns_id) {
+                let proof = NsProof::new(block.payload(), &ns_index, common.common()).context(
+                    CustomSnafu {
+                        message: format!("failed to make proof for namespace {ns_id}"),
+                        status: StatusCode::NOT_FOUND,
+                    },
+                )?;
+
+                Ok(NamespaceProofQueryData {
+                    transactions: proof.export_all_txs(&ns_id),
+                    proof: Some(proof),
+                })
             } else {
-                Vec::new()
-            };
-
-            Ok(NamespaceProofQueryData {
-                transactions,
-                proof,
-            })
+                // ns_id not found in ns_table
+                Ok(NamespaceProofQueryData {
+                    proof: None,
+                    transactions: Vec::new(),
+                })
+            }
         }
         .boxed()
     })?;
@@ -178,16 +167,6 @@ where
                 .body_auto::<Transaction, Ver>(Ver::instance())
                 .map_err(Error::from_request_error)?;
 
-            // Transactions with namespaces that do not fit in the u32
-            // cannot be included in the block.
-            // TODO: This issue will be addressed in the next release.
-            if tx.namespace() > NamespaceId::from(u32::MAX as u64) {
-                return Err(Error::Custom {
-                    message: "Transaction namespace > u32::MAX".to_string(),
-                    status: StatusCode::BAD_REQUEST,
-                });
-            }
-
             let hash = tx.commit();
             state
                 .read(|state| state.submit(tx).boxed())

sequencer/src/bin/nasty-client.rs

@@ -31,14 +31,10 @@ use hotshot_query_service::{
     metrics::PrometheusMetrics,
     node::TimeWindowQueryData,
 };
-use hotshot_types::{
-    traits::metrics::{Counter, Gauge, Histogram, Metrics as _},
-    vid::{vid_scheme, VidSchemeType},
-};
+use hotshot_types::traits::metrics::{Counter, Gauge, Histogram, Metrics as _};
 use jf_merkle_tree::{
     ForgetableMerkleTreeScheme, MerkleCommitment, MerkleTreeScheme, UniversalMerkleTreeScheme,
 };
-use jf_vid::VidScheme;
 use rand::{seq::SliceRandom, RngCore};
 use sequencer::{
     api::endpoints::NamespaceProofQueryData,
@@ -1015,11 +1011,13 @@ impl ResourceManager<BlockQueryData<SeqTypes>> {
                 self.get(format!("availability/header/{block}")).await
             })
             .await?;
-        if header.ns_table.is_empty() {
+        let num_namespaces = header.ns_table.iter().count();
+        if num_namespaces == 0 {
             tracing::info!("not fetching namespace because block {block} is empty");
             return Ok(());
         }
-        let ns = header.ns_table.get_table_entry(index).0;
+        let ns_index = header.ns_table.iter().nth(index % num_namespaces).unwrap();
+        let ns = header.ns_table.read_ns_id(&ns_index).unwrap();
 
         let ns_proof: NamespaceProofQueryData = self
             .retry(info_span!("fetch namespace", %ns), || async {
@@ -1034,13 +1032,21 @@ impl ResourceManager<BlockQueryData<SeqTypes>> {
                 self.get(format!("availability/vid/common/{block}")).await
             })
             .await?;
-        let vid = vid_scheme(VidSchemeType::get_num_storage_nodes(vid_common.common()) as usize);
+        ensure!(
+            ns_proof.proof.is_some(),
+            format!("missing namespace proof for {block}:{ns}")
+        );
         ensure!(
             ns_proof
                 .proof
-                .verify(&vid, &header.payload_commitment, &header.ns_table)
+                .unwrap()
+                .verify(
+                    &header.ns_table,
+                    &header.payload_commitment,
+                    vid_common.common()
+                )
                 .is_some(),
-            format!("namespace proof for {block}:{ns} is invalid")
+            format!("failure to verify namespace proof for {block}:{ns}")
         );
 
         self.metrics.query_namespace_actions.add(1);

sequencer/src/bin/submit-transactions.rs

@@ -60,15 +60,15 @@ struct Options {
         default_value = "10000",
         env = "ESPRESSO_SUBMIT_TRANSACTIONS_MIN_NAMESPACE"
     )]
-    min_namespace: u64,
+    min_namespace: u32,
 
     /// Maximum namespace ID to submit to.
     #[clap(
         long,
         default_value = "10010",
         env = "ESPRESSO_SUBMIT_TRANSACTIONS_MAX_NAMESPACE"
     )]
-    max_namespace: u64,
+    max_namespace: u32,
 
     /// Mean delay between submitting transactions.
     ///
@@ -327,6 +327,8 @@ async fn server<Ver: StaticVersionType + 'static>(port: u16, bind_version: Ver)
 }
 
 fn random_transaction(opt: &Options, rng: &mut ChaChaRng) -> Transaction {
+    // TODO instead use NamespaceId::random, but that does not allow us to
+    // enforce `gen_range(opt.min_namespace..=opt.max_namespace)`
     let namespace = rng.gen_range(opt.min_namespace..=opt.max_namespace);
 
     let len = rng.gen_range(opt.min_size..=opt.max_size);