introduce mmr stress test

.github/workflows/ci.yml

@@ -6,6 +6,9 @@ on:
       - main
   pull_request_target:
     types: [ opened, synchronize ]
+#  pull_request:
+#    branches:
+#      - main
 
 concurrency:
   group: ci-${{ github.head_ref || github.ref_name }}
@@ -255,6 +258,15 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
 
+      - name: MMR gadget integration tests
+        run: |
+          # build hyperbridge binary
+          cargo build -p hyperbridge --release
+          ./target/release/hyperbridge simnode --chain=gargantua-2000 --name=alice --tmp  --state-pruning=archive --blocks-pruning=archive --rpc-port=9990 --port 40337 --log="mmr=trace" --rpc-cors=all --unsafe-rpc-external --rpc-methods=unsafe &
+          ./scripts/wait_for_tcp_port_opening.sh localhost 9990
+          cargo test -p mmr-simnode-tests -- --nocapture --ignored
+          kill -9 $!
+
       - name: Run Parachain Devnet
         run: |
           # install polkadot binary
@@ -268,12 +280,8 @@ jobs:
           sudo chmod +x ./polkadot-execute-worker
           sudo chmod +x ./polkadot-prepare-worker
 
-          cd ../../../
-          ls -la polkadot-sdk/target/release
-
-          # build hyperbridge binary
-          cd hyperbridge
-          cargo build -p hyperbridge --release
+          # get back in to hyperbridge
+          cd ../../../hyperbridge
 
           # install zombienet
           wget -O ./zombienet -q --show-progress https://github.com/paritytech/zombienet/releases/download/v1.3.89/zombienet-linux-x64
@@ -312,14 +320,6 @@ jobs:
         run: |
           cargo test -p bsc-prover -- --nocapture --ignored
 
-      - name: Mmr gadget tests
-        run: | 
-          cargo build -p hyperbridge --release --features simnode
-          ./target/release/hyperbridge simnode --chain gargantua-2000 --name=alice --tmp  --state-pruning=archive --blocks-pruning=archive --rpc-port=9990 --port 40337 --log="mmr=trace" --unsafe-rpc-external --rpc-methods=unsafe &
-          sleep 20;
-          cargo test -p simnode-tests -- --nocapture --ignored
-          kill -9 $!
-
   check-solidity:
     name: Check ismp-solidity
     runs-on: ubuntu-latest

Cargo.toml

@@ -56,7 +56,7 @@ members = [
     "modules/client",
 
     # simnode
-    "modules/trees/mmr/simnode-tests",
+    "modules/trees/mmr/tests",
     # subxt subxt-utils
     "modules/subxt/subxt-utils"
 ]
@@ -279,7 +279,7 @@ nexus-runtime = { path = "./parachain/runtimes/nexus", default-features = false
 # Simnode
 sc-simnode = { version = "1.6.1" }
 simnode-runtime-api = { version = "1.6.0", default-features = false }
-simnode-tests = { path = "modules/trees/mmr/simnode-tests" }
+simnode-tests = { path = "modules/trees/mmr/tests" }
 
 # Subxt
 subxt = { version = "0.30.1", default-features = false }

modules/trees/mmr/primitives/src/lib.rs

@@ -167,7 +167,7 @@ impl<H: traits::Hash, L: FullLeaf> DataOrHash<H, L> {
 }
 
 /// Converts a node's mmr position, to it's k-index. The k-index is the node's index within a layer
-/// of the subtree.
+/// of the subtree. Refer to https://research.polytope.technology/merkle-mountain-range-multi-proofs
 pub fn mmr_position_to_k_index(mut leaves: Vec<u64>, mmr_size: u64) -> Vec<(u64, usize)> {
     let peaks = get_peaks(mmr_size);
     let mut leaves_with_k_indices = vec![];

modules/trees/mmr/simnode-tests/Cargo.toml → modules/trees/mmr/tests/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "simnode-tests"
+name = "mmr-simnode-tests"
 version = "0.1.0"
 edition = "2021"
 description = "Simnode tests for mmr gadget"
@@ -12,6 +12,7 @@ subxt = { workspace = true, default-features = true }
 anyhow = "1.0.70"
 futures = "0.3.28"
 tokio = { version = "1.27.0", features = ["macros"] }
+indicatif = "0.17.8"
 
 codec = { workspace = true, features = ["derive"], default-features = true }
 sp-core = { workspace = true, features = ["full_crypto", "serde"], default-features = true }
@@ -26,3 +27,6 @@ mmr-primitives = { workspace = true, default-features = true }
 pallet-mmr = { workspace = true, default-features = true }
 pallet-ismp-rpc = { workspace = true }
 merkle-mountain-range = { workspace = true }
+
+[features]
+stress-test = []

modules/trees/mmr/simnode-tests/src/lib.rs → modules/trees/mmr/tests/src/lib.rs

@@ -355,3 +355,58 @@ async fn dispatch_requests() -> Result<(), anyhow::Error> {
     assert!(res);
     Ok(())
 }
+
+#[cfg(feature = "stress-test")]
+#[tokio::test]
+async fn test_insert_1_billion_mmr_leaves() -> Result<(), anyhow::Error> {
+    // try to estimate the storage requirements on offchaindb with 1 billion leaves in mmr
+    use indicatif::ProgressBar;
+
+    let port = env::var("PORT").unwrap_or("9990".into());
+    let client = OnlineClient::<Hyperbridge>::from_url(format!("ws://127.0.0.1:{}", port)).await?;
+    let pb = ProgressBar::new(1_000_000);
+    for _ in 0..100_000 {
+        // Initialize MMR Pallet by dispatching some leaves and finalizing
+        let params = EvmParams {
+            module: H160::random(),
+            destination: Ethereum::ExecutionLayer,
+            timeout: 0,
+            count: 10_000,
+        };
+        let call = client
+            .tx()
+            .call_data(&gargantua::api::tx().ismp_demo().dispatch_to_evm(params))?;
+        let _account_id = AsRef::<[u8; 32]>::as_ref(&Keyring::Ferdie.to_account_id()).clone();
+
+        let _ = client
+            .rpc()
+            .request::<CreatedBlock<H256>>("engine_createBlock", rpc_params![true, false])
+            .await?;
+
+        let extrinsic: Bytes = client
+            .rpc()
+            .request(
+                "simnode_authorExtrinsic",
+                // author an extrinsic from alice
+                rpc_params![Bytes::from(call), Keyring::Ferdie.to_account_id().to_ss58check()],
+            )
+            .await?;
+        SubmittableExtrinsic::from_bytes(client.clone(), extrinsic.0).submit().await?;
+
+        let created_block = client
+            .rpc()
+            .request::<CreatedBlock<H256>>("engine_createBlock", rpc_params![true, false])
+            .await?;
+
+        // Finalize a new block so that we are sure mmr gadget gets the notification
+        let _ = client
+            .rpc()
+            .request::<bool>("engine_finalizeBlock", rpc_params![created_block.hash])
+            .await?;
+        pb.inc(1);
+    }
+
+    pb.finish_with_message("Inserted 1 billion leaves");
+
+    Ok(())
+}

parachain/node/Cargo.toml

@@ -85,10 +85,10 @@ cumulus-client-consensus-proposer = { workspace = true }
 cumulus-client-collator = { workspace = true }
 
 # Simnode
-sc-simnode = { workspace = true, optional = true }
-frame-system = { workspace = true, optional = true }
-pallet-transaction-payment = { workspace = true, optional = true }
-simnode-runtime-api = { workspace = true, optional = true }
+sc-simnode = { workspace = true, features = ["parachain"] }
+frame-system = { workspace = true }
+pallet-transaction-payment = { workspace = true }
+simnode-runtime-api = { workspace = true }
 
 [build-dependencies]
 substrate-build-script-utils = { version = "11.0.0" }
@@ -99,9 +99,9 @@ runtime-benchmarks = [
 	"try-runtime-cli/try-runtime",
 	"gargantua-runtime/runtime-benchmarks",
 	"messier-runtime/runtime-benchmarks",
+	"nexus-runtime/runtime-benchmarks",
 	"polkadot-cli/runtime-benchmarks",
 ]
-
 try-runtime = [
 	"try-runtime-cli/try-runtime",
 	"gargantua-runtime/try-runtime",
@@ -111,13 +111,3 @@ async-backing = [
 	"gargantua-runtime/async-backing",
 	"messier-runtime/async-backing"
 ]
-
-simnode = [
-	"sc-simnode/parachain",
-	"frame-system",
-	"pallet-transaction-payment",
-	"simnode-runtime-api",
-	"gargantua-runtime/simnode",
-	"nexus-runtime/simnode",
-	"messier-runtime/simnode"
-]

parachain/node/src/cli.rs

@@ -59,7 +59,6 @@ pub enum Subcommand {
     TryRuntime,
 
     /// Runs the node with signature verification override and manual seal.
-    #[cfg(feature = "simnode")]
     Simnode(sc_simnode::cli::SimnodeCli),
 }