[stable 2409] Backport #5741

Cargo.toml

@@ -1313,7 +1313,7 @@ tikv-jemalloc-ctl = { version = "0.5.0" }
 tikv-jemallocator = { version = "0.5.0" }
 time = { version = "0.3" }
 tiny-keccak = { version = "2.0.2" }
-tokio = { version = "1.37.0", default-features = false }
+tokio = { version = "1.40.0", default-features = false }
 tokio-retry = { version = "0.3.0" }
 tokio-stream = { version = "0.1.14" }
 tokio-test = { version = "0.4.2" }

prdoc/pr_5741.prdoc

@@ -0,0 +1,23 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: make RPC endpoint `chainHead_v1_storage` faster
+
+doc:
+  - audience: Node Operator
+    description: |
+      The RPC endpoint `chainHead_v1_storage` now relies solely on backpressure to
+      determine how quickly to serve back values instead of handing back a fixed number
+      of entries and then expecting the client to ask for more. This should improve the 
+      throughput for bigger storage queries significantly.
+
+      Benchmarks using subxt on localhost:
+        - Iterate over 10 accounts on westend-dev -> ~2-3x faster
+        - Fetch 1024 storage values (i.e, not descedant values) -> ~50x faster
+        - Fetch 1024 descendant values -> ~500x faster
+
+crates: 
+  - name: sc-rpc-spec-v2
+    bump: major
+  - name: sc-rpc-server
+    bump: patch

substrate/client/rpc-servers/src/lib.rs

@@ -224,8 +224,9 @@ where
 							),
 						};
 
-						let rpc_middleware =
-							RpcServiceBuilder::new().option_layer(middleware_layer.clone());
+						let rpc_middleware = RpcServiceBuilder::new()
+							.rpc_logger(1024)
+							.option_layer(middleware_layer.clone());
 						let mut svc = service_builder
 							.set_rpc_middleware(rpc_middleware)
 							.build(methods, stop_handle);

substrate/client/rpc-spec-v2/Cargo.toml

@@ -17,28 +17,18 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 jsonrpsee = { workspace = true, features = ["client-core", "macros", "server-core"] }
-sc-chain-spec.workspace = true
-sc-chain-spec.default-features = true
-sc-transaction-pool-api.workspace = true
-sc-transaction-pool-api.default-features = true
-sp-core.workspace = true
-sp-core.default-features = true
-sp-runtime.workspace = true
-sp-runtime.default-features = true
-sp-api.workspace = true
-sp-api.default-features = true
-sp-rpc.workspace = true
-sp-rpc.default-features = true
-sp-blockchain.workspace = true
-sp-blockchain.default-features = true
-sp-version.workspace = true
-sp-version.default-features = true
-sc-client-api.workspace = true
-sc-client-api.default-features = true
-sc-utils.workspace = true
-sc-utils.default-features = true
-sc-rpc.workspace = true
-sc-rpc.default-features = true
+# Internal chain structures for "chain_spec".
+sc-chain-spec = { workspace = true, default-features = true }
+# Pool for submitting extrinsics required by "transaction"
+sc-transaction-pool-api = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
+sp-runtime = { workspace = true, default-features = true }
+sp-api = { workspace = true, default-features = true }
+sp-rpc = { workspace = true, default-features = true }
+sp-blockchain = { workspace = true, default-features = true }
+sp-version = { workspace = true, default-features = true }
+sc-client-api = { workspace = true, default-features = true }
+sc-rpc = { workspace = true, default-features = true }
 codec = { workspace = true, default-features = true }
 thiserror = { workspace = true }
 serde = { workspace = true, default-features = true }
@@ -57,14 +47,16 @@ schnellru = { workspace = true }
 jsonrpsee = { workspace = true, features = ["server", "ws-client"] }
 serde_json = { workspace = true, default-features = true }
 tokio = { features = ["macros"], workspace = true, default-features = true }
-substrate-test-runtime-client = { path = "../../test-utils/runtime/client" }
-substrate-test-runtime = { path = "../../test-utils/runtime" }
-substrate-test-runtime-transaction-pool = { path = "../../test-utils/runtime/transaction-pool" }
-sp-consensus = { default-features = true, path = "../../primitives/consensus/common" }
-sp-externalities = { default-features = true, path = "../../primitives/externalities" }
-sp-maybe-compressed-blob = { default-features = true, path = "../../primitives/maybe-compressed-blob" }
-sc-block-builder = { default-features = true, path = "../block-builder" }
-sc-service = { features = ["test-helpers"], default-features = true, path = "../service" }
+substrate-test-runtime-client = { workspace = true }
+substrate-test-runtime = { workspace = true }
+substrate-test-runtime-transaction-pool = { workspace = true }
+sp-consensus = { workspace = true, default-features = true }
+sp-externalities = { workspace = true, default-features = true }
+sp-maybe-compressed-blob = { workspace = true, default-features = true }
+sc-block-builder = { workspace = true, default-features = true }
+sc-service = { features = ["test-helpers"], workspace = true, default-features = true }
+sc-rpc = { workspace = true, default-features = true, features = ["test-helpers"] }
 assert_matches = { workspace = true }
 pretty_assertions = { workspace = true }
-sc-transaction-pool = { default-features = true, path = "../transaction-pool" }
+sc-transaction-pool = { workspace = true, default-features = true }
+sc-utils = { workspace = true, default-features = true }

substrate/client/rpc-spec-v2/src/archive/archive.rs

@@ -275,6 +275,7 @@ where
 			self.storage_max_descendant_responses,
 			self.storage_max_queried_items,
 		);
+
 		Ok(storage_client.handle_query(hash, items, child_trie))
 	}
 }