Merge branch 'master' into ao-past-session-slashing-runtime

* master:
  XCM: Tools for uniquely referencing messages (#7234)
  Companion: Substrate#13869 (#7119)
  Companion for Substrate#14214 (#7283)
  Fix flaky test and error reporting (#7282)
  impl guide: Update Collator Generation (#7250)
  Add staking-miner bin (#7273)
  metrics: tests: Fix flaky runtime_can_publish_metrics (#7279)

node/collation-generation/src/lib.rs

@@ -15,6 +15,17 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 //! The collation generation subsystem is the interface between polkadot and the collators.
+//!
+//! # Protocol
+//!
+//! On every `ActiveLeavesUpdate`:
+//!
+//! * If there is no collation generation config, ignore.
+//! * Otherwise, for each `activated` head in the update:
+//!   * Determine if the para is scheduled on any core by fetching the `availability_cores` Runtime API.
+//!   * Use the Runtime API subsystem to fetch the full validation data.
+//!   * Invoke the `collator`, and use its outputs to produce a [`CandidateReceipt`], signed with the configuration's `key`.
+//!   * Dispatch a [`CollatorProtocolMessage::DistributeCollation`](receipt, pov)`.
 
 #![deny(missing_docs)]
 

node/core/dispute-coordinator/src/initialized.rs

@@ -214,62 +214,61 @@ impl Initialized {
 			gum::trace!(target: LOG_TARGET, "Waiting for message");
 			let mut overlay_db = OverlayedBackend::new(backend);
 			let default_confirm = Box::new(|| Ok(()));
-			let confirm_write = match MuxedMessage::receive(ctx, &mut self.participation_receiver)
-				.await?
-			{
-				MuxedMessage::Participation(msg) => {
-					gum::trace!(target: LOG_TARGET, "MuxedMessage::Participation");
-					let ParticipationStatement {
-						session,
-						candidate_hash,
-						candidate_receipt,
-						outcome,
-					} = self.participation.get_participation_result(ctx, msg).await?;
-					if let Some(valid) = outcome.validity() {
-						gum::trace!(
-							target: LOG_TARGET,
-							?session,
-							?candidate_hash,
-							?valid,
-							"Issuing local statement based on participation outcome."
-						);
-						self.issue_local_statement(
-							ctx,
-							&mut overlay_db,
+			let confirm_write =
+				match MuxedMessage::receive(ctx, &mut self.participation_receiver).await? {
+					MuxedMessage::Participation(msg) => {
+						gum::trace!(target: LOG_TARGET, "MuxedMessage::Participation");
+						let ParticipationStatement {
+							session,
 							candidate_hash,
 							candidate_receipt,
-							session,
-							valid,
-							clock.now(),
-						)
-						.await?;
-					} else {
-						gum::warn!(target: LOG_TARGET, ?outcome, "Dispute participation failed");
-					}
-					default_confirm
-				},
-				MuxedMessage::Subsystem(msg) => match msg {
-					FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()),
-					FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
-						gum::trace!(target: LOG_TARGET, "OverseerSignal::ActiveLeaves");
-						self.process_active_leaves_update(
-							ctx,
-							&mut overlay_db,
-							update,
-							clock.now(),
-						)
-						.await?;
+							outcome,
+						} = self.participation.get_participation_result(ctx, msg).await?;
+						if let Some(valid) = outcome.validity() {
+							gum::trace!(
+								target: LOG_TARGET,
+								?session,
+								?candidate_hash,
+								?valid,
+								"Issuing local statement based on participation outcome."
+							);
+							self.issue_local_statement(
+								ctx,
+								&mut overlay_db,
+								candidate_hash,
+								candidate_receipt,
+								session,
+								valid,
+								clock.now(),
+							)
+							.await?;
+						} else {
+							gum::warn!(target: LOG_TARGET, ?outcome, "Dispute participation failed");
+						}
 						default_confirm
 					},
-					FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, n)) => {
-						gum::trace!(target: LOG_TARGET, "OverseerSignal::BlockFinalized");
-						self.scraper.process_finalized_block(&n);
-						default_confirm
+					MuxedMessage::Subsystem(msg) => match msg {
+						FromOrchestra::Signal(OverseerSignal::Conclude) => return Ok(()),
+						FromOrchestra::Signal(OverseerSignal::ActiveLeaves(update)) => {
+							gum::trace!(target: LOG_TARGET, "OverseerSignal::ActiveLeaves");
+							self.process_active_leaves_update(
+								ctx,
+								&mut overlay_db,
+								update,
+								clock.now(),
+							)
+							.await?;
+							default_confirm
+						},
+						FromOrchestra::Signal(OverseerSignal::BlockFinalized(_, n)) => {
+							gum::trace!(target: LOG_TARGET, "OverseerSignal::BlockFinalized");
+							self.scraper.process_finalized_block(&n);
+							default_confirm
+						},
+						FromOrchestra::Communication { msg } =>
+							self.handle_incoming(ctx, &mut overlay_db, msg, clock.now()).await?,
 					},
-					FromOrchestra::Communication { msg } =>
-						self.handle_incoming(ctx, &mut overlay_db, msg, clock.now()).await?,
-				},
-			};
+				};
 
 			if !overlay_db.is_empty() {
 				let ops = overlay_db.into_write_ops();

node/metrics/src/tests.rs

@@ -61,8 +61,8 @@ async fn runtime_can_publish_metrics() {
 	// Start validator Bob.
 	let _bob = run_validator_node(bob_config, None);
 
-	// Wait for Alice to author two blocks.
-	alice.wait_for_blocks(2).await;
+	// Wait for Alice to see two finalized blocks.
+	alice.wait_for_finalized_blocks(2).await;
 
 	let metrics_uri = format!("http://localhost:{}/metrics", DEFAULT_PROMETHEUS_PORT);
 	let metrics = scrape_prometheus_metrics(&metrics_uri).await;

node/network/availability-recovery/src/tests.rs

@@ -172,7 +172,9 @@ async fn overseer_signal(
 		.send(FromOrchestra::Signal(signal))
 		.timeout(TIMEOUT)
 		.await
-		.expect("10ms is more than enough for sending signals.");
+		.unwrap_or_else(|| {
+			panic!("{}ms is more than enough for sending signals.", TIMEOUT.as_millis())
+		});
 }
 
 async fn overseer_send(
@@ -184,7 +186,9 @@ async fn overseer_send(
 		.send(FromOrchestra::Communication { msg })
 		.timeout(TIMEOUT)
 		.await
-		.expect("10ms is more than enough for sending messages.");
+		.unwrap_or_else(|| {
+			panic!("{}ms is more than enough for sending messages.", TIMEOUT.as_millis())
+		});
 }
 
 async fn overseer_recv(

node/network/bitfield-distribution/src/tests.rs

@@ -41,11 +41,12 @@ use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
 
 use std::{iter::FromIterator as _, sync::Arc, time::Duration};
 
+const TIMEOUT: Duration = Duration::from_millis(50);
 macro_rules! launch {
 	($fut:expr) => {
-		$fut.timeout(Duration::from_millis(10))
-			.await
-			.expect("10ms is more than enough for sending messages.")
+		$fut.timeout(TIMEOUT).await.unwrap_or_else(|| {
+			panic!("{}ms is more than enough for sending messages.", TIMEOUT.as_millis())
+		});
 	};
 }
 
@@ -220,7 +221,7 @@ fn receive_invalid_signature() {
 		));
 
 		// reputation doesn't change due to one_job_per_validator check
-		assert!(handle.recv().timeout(Duration::from_millis(10)).await.is_none());
+		assert!(handle.recv().timeout(TIMEOUT).await.is_none());
 
 		launch!(handle_network_msg(
 			&mut ctx,
@@ -523,7 +524,7 @@ fn do_not_relay_message_twice() {
 		);
 
 		// There shouldn't be any other message
-		assert!(handle.recv().timeout(Duration::from_millis(10)).await.is_none());
+		assert!(handle.recv().timeout(TIMEOUT).await.is_none());
 	});
 }
 

node/test/service/src/lib.rs

@@ -21,7 +21,7 @@
 pub mod chain_spec;
 
 pub use chain_spec::*;
-use futures::future::Future;
+use futures::{future::Future, stream::StreamExt};
 use polkadot_node_primitives::{CollationGenerationConfig, CollatorFn};
 use polkadot_node_subsystem::messages::{CollationGenerationMessage, CollatorProtocolMessage};
 use polkadot_overseer::Handle;
@@ -35,8 +35,9 @@ use polkadot_test_runtime::{
 	ParasCall, ParasSudoWrapperCall, Runtime, SignedExtra, SignedPayload, SudoCall,
 	UncheckedExtrinsic, VERSION,
 };
+
 use sc_chain_spec::ChainSpec;
-use sc_client_api::execution_extensions::ExecutionStrategies;
+use sc_client_api::{execution_extensions::ExecutionStrategies, BlockchainEvents};
 use sc_network::{
 	config::{NetworkConfiguration, TransportConfig},
 	multiaddr, NetworkStateInfo,
@@ -54,14 +55,14 @@ use sp_keyring::Sr25519Keyring;
 use sp_runtime::{codec::Encode, generic, traits::IdentifyAccount, MultiSigner};
 use sp_state_machine::BasicExternalities;
 use std::{
+	collections::HashSet,
 	net::{Ipv4Addr, SocketAddr},
 	path::PathBuf,
 	sync::Arc,
 };
 use substrate_test_client::{
 	BlockchainEventsExt, RpcHandlersExt, RpcTransactionError, RpcTransactionOutput,
 };
-
 /// Declare an instance of the native executor named `PolkadotTestExecutorDispatch`. Include the wasm binary as the
 /// equivalent wasm code.
 pub struct PolkadotTestExecutorDispatch;
@@ -335,6 +336,20 @@ impl PolkadotTestNode {
 		self.client.wait_for_blocks(count)
 	}
 
+	/// Wait for `count` blocks to be finalized and then exit. Similarly with `wait_for_blocks` this function will
+	/// not return if no block are ever finalized.
+	pub async fn wait_for_finalized_blocks(&self, count: usize) {
+		let mut import_notification_stream = self.client.finality_notification_stream();
+		let mut blocks = HashSet::new();
+
+		while let Some(notification) = import_notification_stream.next().await {
+			blocks.insert(notification.hash);
+			if blocks.len() == count {
+				break
+			}
+		}
+	}
+
 	/// Register the collator functionality in the overseer of this node.
 	pub async fn register_collator(
 		&mut self,

roadmap/implementers-guide/src/node/collators/README.md

@@ -1,3 +1,6 @@
 # Collators
 
 Collators are special nodes which bridge a parachain to the relay chain. They are simultaneously full nodes of the parachain, and at least light clients of the relay chain. Their overall contribution to the system is the generation of Proofs of Validity for parachain candidates.
+
+The **Collation Generation** subsystem triggers collators to produce collations
+and then forwards them to **Collator Protocol** to circulate to validators.

roadmap/implementers-guide/src/node/collators/collation-generation.md

@@ -4,17 +4,32 @@ The collation generation subsystem is executed on collator nodes and produces ca
 
 ## Protocol
 
-Input: `CollationGenerationMessage`
+Collation generation for Parachains currently works in the following way:
 
-```rust
-enum CollationGenerationMessage {
-  Initialize(CollationGenerationConfig),
-}
-```
+1.  A new relay chain block is imported.
+2.  The collation generation subsystem checks if the core associated to
+    the parachain is free and if yes, continues.
+3.  Collation generation calls our collator callback to generate a PoV.
+4.  Authoring logic determines if the current node should build a PoV.
+5.  Build new PoV and give it back to collation generation.
+
+## Messages
 
-No more than one initialization message should ever be sent to the collation generation subsystem.
+### Incoming
 
-Output: `CollationDistributionMessage`
+- `ActiveLeaves`
+  - Notification of a change in the set of active leaves.
+  - Triggers collation generation procedure outlined in "Protocol" section.
+- `CollationGenerationMessage::Initialize`
+  - Initializes the subsystem. Carries a config.
+  - No more than one initialization message should ever be sent to the collation
+    generation subsystem.
+  - Sent by a collator to initialize this subsystem.
+
+### Outgoing
+
+- `CollatorProtocolMessage::DistributeCollation`
+  - Provides a generated collation to distribute to validators.
 
 ## Functionality
 
@@ -94,15 +109,34 @@ pub struct CollationGenerationConfig {
 
 The configuration should be optional, to allow for the case where the node is not run with the capability to collate.
 
-On `ActiveLeavesUpdate`:
+### Summary in plain English
+
+- **Collation (output of a collator)**
+
+  - Contains the PoV (proof to verify the state transition of the
+    parachain) and other data.
+
+- **Collation result**
+
+  - Contains the collation, and an optional result sender for a
+    collation-seconded signal.
+
+- **Collation seconded signal**
+
+  - The signal that is returned when a collation was seconded by a
+    validator.
+
+- **Collation function**
+
+  - Called with the relay chain block the parablock will be built on top
+    of.
+  - Called with the validation data.
+    - Provides information about the state of the parachain on the relay
+      chain.
+
+- **Collation generation config**
 
-* If there is no collation generation config, ignore.
-* Otherwise, for each `activated` head in the update:
-  * Determine if the para is scheduled on any core by fetching the `availability_cores` Runtime API.
-  * Determine an occupied core assumption to make about the para. Scheduled cores can make `OccupiedCoreAssumption::Free`.
-  * Use the Runtime API subsystem to fetch the full validation data.
-  * Invoke the `collator`, and use its outputs to produce a `CandidateReceipt`, signed with the configuration's `key`.
-  * Dispatch a [`CollatorProtocolMessage`][CPM]`::DistributeCollation(receipt, pov)`.
+  - Contains collator's authentication key, collator function, and
+    parachain ID.
 
 [CP]: collator-protocol.md
-[CPM]: ../../types/overseer-protocol.md#collatorprotocolmessage

runtime/common/src/assigned_slots.rs

@@ -631,7 +631,7 @@ mod tests {
 		type MaxLocks = ();
 		type MaxReserves = ();
 		type ReserveIdentifier = [u8; 8];
-		type HoldIdentifier = ();
+		type RuntimeHoldReason = RuntimeHoldReason;
 		type FreezeIdentifier = ();
 		type MaxHolds = ConstU32<1>;
 		type MaxFreezes = ConstU32<1>;

runtime/common/src/auctions.rs

@@ -748,7 +748,7 @@ mod tests {
 		type MaxLocks = ();
 		type MaxReserves = MaxReserves;
 		type ReserveIdentifier = [u8; 8];
-		type HoldIdentifier = ();
+		type RuntimeHoldReason = RuntimeHoldReason;
 		type FreezeIdentifier = ();
 		type MaxHolds = ConstU32<1>;
 		type MaxFreezes = ConstU32<1>;

runtime/common/src/claims.rs

@@ -787,7 +787,7 @@ mod tests {
 		type MaxReserves = ();
 		type ReserveIdentifier = [u8; 8];
 		type WeightInfo = ();
-		type HoldIdentifier = ();
+		type RuntimeHoldReason = RuntimeHoldReason;
 		type FreezeIdentifier = ();
 		type MaxHolds = ConstU32<1>;
 		type MaxFreezes = ConstU32<1>;

runtime/common/src/crowdloan/mod.rs

@@ -945,7 +945,7 @@ mod tests {
 		type MaxReserves = ();
 		type ReserveIdentifier = [u8; 8];
 		type WeightInfo = ();
-		type HoldIdentifier = ();
+		type RuntimeHoldReason = RuntimeHoldReason;
 		type FreezeIdentifier = ();
 		type MaxHolds = ConstU32<1>;
 		type MaxFreezes = ConstU32<1>;

runtime/common/src/impls.rs

@@ -186,7 +186,7 @@ mod tests {
 		type MaxReserves = ();
 		type ReserveIdentifier = [u8; 8];
 		type WeightInfo = ();
-		type HoldIdentifier = ();
+		type RuntimeHoldReason = RuntimeHoldReason;
 		type FreezeIdentifier = ();
 		type MaxHolds = ConstU32<1>;
 		type MaxFreezes = ConstU32<1>;