Merge branch 'master' into tsv-no-db-leaves

node/core/dispute-coordinator/src/backend.rs

@@ -21,7 +21,6 @@
 //! [`Backend`], maintaining consistency between queries and temporary writes,
 //! before any commit to the underlying storage is made.
 
-use polkadot_node_subsystem::SubsystemResult;
 use polkadot_primitives::{CandidateHash, SessionIndex};
 
 use std::collections::HashMap;
@@ -40,17 +39,17 @@ pub enum BackendWriteOp {
 /// An abstraction over backend storage for the logic of this subsystem.
 pub trait Backend {
 	/// Load the earliest session, if any.
-	fn load_earliest_session(&self) -> SubsystemResult<Option<SessionIndex>>;
+	fn load_earliest_session(&self) -> FatalResult<Option<SessionIndex>>;
 
 	/// Load the recent disputes, if any.
-	fn load_recent_disputes(&self) -> SubsystemResult<Option<RecentDisputes>>;
+	fn load_recent_disputes(&self) -> FatalResult<Option<RecentDisputes>>;
 
 	/// Load the candidate votes for the specific session-candidate pair, if any.
 	fn load_candidate_votes(
 		&self,
 		session: SessionIndex,
 		candidate_hash: &CandidateHash,
-	) -> SubsystemResult<Option<CandidateVotes>>;
+	) -> FatalResult<Option<CandidateVotes>>;
 
 	/// Atomically writes the list of operations, with later operations taking precedence over
 	/// prior.
@@ -93,7 +92,7 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
 	}
 
 	/// Load the earliest session, if any.
-	pub fn load_earliest_session(&self) -> SubsystemResult<Option<SessionIndex>> {
+	pub fn load_earliest_session(&self) -> FatalResult<Option<SessionIndex>> {
 		if let Some(val) = self.earliest_session {
 			return Ok(Some(val))
 		}
@@ -102,7 +101,7 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
 	}
 
 	/// Load the recent disputes, if any.
-	pub fn load_recent_disputes(&self) -> SubsystemResult<Option<RecentDisputes>> {
+	pub fn load_recent_disputes(&self) -> FatalResult<Option<RecentDisputes>> {
 		if let Some(val) = &self.recent_disputes {
 			return Ok(Some(val.clone()))
 		}
@@ -115,7 +114,7 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
 		&self,
 		session: SessionIndex,
 		candidate_hash: &CandidateHash,
-	) -> SubsystemResult<Option<CandidateVotes>> {
+	) -> FatalResult<Option<CandidateVotes>> {
 		if let Some(val) = self.candidate_votes.get(&(session, *candidate_hash)) {
 			return Ok(val.clone())
 		}

node/core/dispute-coordinator/src/db/v1.rs

@@ -17,7 +17,6 @@
 //! `V1` database for the dispute coordinator.
 
 use polkadot_node_primitives::DisputeStatus;
-use polkadot_node_subsystem::{SubsystemError, SubsystemResult};
 use polkadot_node_subsystem_util::database::{DBTransaction, Database};
 use polkadot_primitives::{
 	CandidateHash, CandidateReceipt, Hash, InvalidDisputeStatementKind, SessionIndex,
@@ -109,12 +108,12 @@ impl DbBackend {
 
 impl Backend for DbBackend {
 	/// Load the earliest session, if any.
-	fn load_earliest_session(&self) -> SubsystemResult<Option<SessionIndex>> {
+	fn load_earliest_session(&self) -> FatalResult<Option<SessionIndex>> {
 		load_earliest_session(&*self.inner, &self.config)
 	}
 
 	/// Load the recent disputes, if any.
-	fn load_recent_disputes(&self) -> SubsystemResult<Option<RecentDisputes>> {
+	fn load_recent_disputes(&self) -> FatalResult<Option<RecentDisputes>> {
 		load_recent_disputes(&*self.inner, &self.config)
 	}
 
@@ -123,7 +122,7 @@ impl Backend for DbBackend {
 		&self,
 		session: SessionIndex,
 		candidate_hash: &CandidateHash,
-	) -> SubsystemResult<Option<CandidateVotes>> {
+	) -> FatalResult<Option<CandidateVotes>> {
 		load_candidate_votes(&*self.inner, &self.config, session, candidate_hash)
 	}
 
@@ -287,27 +286,27 @@ pub(crate) fn load_candidate_votes(
 	config: &ColumnConfiguration,
 	session: SessionIndex,
 	candidate_hash: &CandidateHash,
-) -> SubsystemResult<Option<CandidateVotes>> {
+) -> FatalResult<Option<CandidateVotes>> {
 	load_decode(db, config.col_dispute_data, &candidate_votes_key(session, candidate_hash))
-		.map_err(|e| SubsystemError::with_origin("dispute-coordinator", e))
+		.map_err(|e| FatalError::DbReadFailed(e))
 }
 
 /// Load the earliest session, if any.
 pub(crate) fn load_earliest_session(
 	db: &dyn Database,
 	config: &ColumnConfiguration,
-) -> SubsystemResult<Option<SessionIndex>> {
+) -> FatalResult<Option<SessionIndex>> {
 	load_decode(db, config.col_dispute_data, EARLIEST_SESSION_KEY)
-		.map_err(|e| SubsystemError::with_origin("dispute-coordinator", e))
+		.map_err(|e| FatalError::DbReadFailed(e))
 }
 
 /// Load the recent disputes, if any.
 pub(crate) fn load_recent_disputes(
 	db: &dyn Database,
 	config: &ColumnConfiguration,
-) -> SubsystemResult<Option<RecentDisputes>> {
+) -> FatalResult<Option<RecentDisputes>> {
 	load_decode(db, config.col_dispute_data, RECENT_DISPUTES_KEY)
-		.map_err(|e| SubsystemError::with_origin("dispute-coordinator", e))
+		.map_err(|e| FatalError::DbReadFailed(e))
 }
 
 /// Maybe prune data in the DB based on the provided session index.
@@ -321,7 +320,7 @@ pub(crate) fn load_recent_disputes(
 pub(crate) fn note_earliest_session(
 	overlay_db: &mut OverlayedBackend<'_, impl Backend>,
 	new_earliest_session: SessionIndex,
-) -> SubsystemResult<()> {
+) -> FatalResult<()> {
 	match overlay_db.load_earliest_session()? {
 		None => {
 			// First launch - write new-earliest.

node/core/dispute-coordinator/src/initialized.rs

@@ -694,6 +694,10 @@ impl Initialized {
 		Ok(())
 	}
 
+	// We use fatal result rather than result here. Reason being, We for example increase
+	// spam slots in this function. If then the import fails for some non fatal and
+	// unrelated reason, we should likely actually decrement previously incremented spam
+	// slots again, for non fatal errors - which is cumbersome and actually not needed
 	async fn handle_import_statements<Context>(
 		&mut self,
 		ctx: &mut Context,
@@ -702,7 +706,7 @@ impl Initialized {
 		session: SessionIndex,
 		statements: Vec<(SignedDisputeStatement, ValidatorIndex)>,
 		now: Timestamp,
-	) -> Result<ImportStatementsResult> {
+	) -> FatalResult<ImportStatementsResult> {
 		gum::trace!(target: LOG_TARGET, ?statements, "In handle import statements");
 		if !self.rolling_session_window.contains(session) {
 			// It is not valid to participate in an ancient dispute (spam?) or too new.

node/service/Cargo.toml

@@ -74,7 +74,7 @@ serde_json = "1.0.81"
 thiserror = "1.0.31"
 kvdb = "0.13.0"
 kvdb-rocksdb = { version = "0.17.0", optional = true }
-parity-db = { version = "0.4.3", optional = true }
+parity-db = { version = "0.4.4", optional = true }
 
 async-trait = "0.1.57"
 lru = "0.9"

node/subsystem-types/src/messages.rs

@@ -68,7 +68,8 @@ pub enum CandidateBackingMessage {
 	/// given relay-parent (ref. by hash). This candidate must be validated.
 	Second(Hash, CandidateReceipt, PoV),
 	/// Note a validator's statement about a particular candidate. Disagreements about validity must be escalated
-	/// to a broader check by Misbehavior Arbitration. Agreements are simply tallied until a quorum is reached.
+	/// to a broader check by the Disputes Subsystem, though that escalation is deferred until the approval voting
+	/// stage to guarantee availability. Agreements are simply tallied until a quorum is reached.
 	Statement(Hash, SignedFullStatement),
 }
 

node/subsystem-util/Cargo.toml

@@ -34,7 +34,7 @@ sp-application-crypto = { git = "https://github.com/paritytech/substrate", branc
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 kvdb = "0.13.0"
-parity-db = { version = "0.4.3"}
+parity-db = { version = "0.4.4"}
 
 [dev-dependencies]
 assert_matches = "1.4.0"

primitives/src/v2/mod.rs

@@ -1603,6 +1603,9 @@ pub fn supermajority_threshold(n: usize) -> usize {
 }
 
 /// Information about validator sets of a session.
+///
+/// NOTE: `SessionInfo` is frozen. Do not include new fields, consider creating a separate runtime
+/// API. Reasoning and further outlook [here](https://github.com/paritytech/polkadot/issues/6586).
 #[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 #[cfg_attr(feature = "std", derive(PartialEq))]
 pub struct SessionInfo {

roadmap/implementers-guide/src/SUMMARY.md

@@ -63,7 +63,6 @@
     - [Provisioner](node/utility/provisioner.md)
     - [Network Bridge](node/utility/network-bridge.md)
     - [Gossip Support](node/utility/gossip-support.md)
-    - [Misbehavior Arbitration](node/utility/misbehavior-arbitration.md)
     - [Peer Set Manager](node/utility/peer-set-manager.md)
     - [Runtime API Requests](node/utility/runtime-api.md)
     - [Chain API Requests](node/utility/chain-api.md)

roadmap/implementers-guide/src/node/backing/statement-distribution.md

@@ -21,7 +21,7 @@ Output:
 
 Implemented as a gossip protocol. Handle updates to our view and peers' views. Neighbor packets are used to inform peers which chain heads we are interested in data for.
 
-It is responsible for distributing signed statements that we have generated and forwarding them, and for detecting a variety of Validator misbehaviors for reporting to [Misbehavior Arbitration](../utility/misbehavior-arbitration.md). During the Backing stage of the inclusion pipeline, it's the main point of contact with peer nodes. On receiving a signed statement from a peer in the same backing group, assuming the peer receipt state machine is in an appropriate state, it sends the Candidate Receipt to the [Candidate Backing subsystem](candidate-backing.md) to handle the validator's statement. On receiving `StatementDistributionMessage::Share` we make sure to send messages to our backing group in addition to random other peers, to ensure a fast backing process and getting all statements quickly for distribution.
+It is responsible for distributing signed statements that we have generated and forwarding them, and for detecting a variety of Validator misbehaviors for reporting to the [Provisioner Subsystem](../utility/provisioner.md). During the Backing stage of the inclusion pipeline, it's the main point of contact with peer nodes. On receiving a signed statement from a peer in the same backing group, assuming the peer receipt state machine is in an appropriate state, it sends the Candidate Receipt to the [Candidate Backing subsystem](candidate-backing.md) to handle the validator's statement. On receiving `StatementDistributionMessage::Share` we make sure to send messages to our backing group in addition to random other peers, to ensure a fast backing process and getting all statements quickly for distribution.
 
 Track equivocating validators and stop accepting information from them. Establish a data-dependency order:
 

roadmap/implementers-guide/src/node/utility/provisioner.md

@@ -30,7 +30,14 @@ Dispute resolution is complex and is explained in substantially more detail [her
 
 ## Protocol
 
-Input: [`ProvisionerMessage`](../../types/overseer-protocol.md#provisioner-message). Backed candidates come from the [Candidate Backing subsystem](../backing/candidate-backing.md), signed bitfields come from the [Bitfield Distribution subsystem](../availability/bitfield-distribution.md), and misbehavior reports and disputes come from the [Misbehavior Arbitration subsystem](misbehavior-arbitration.md).
+Input: [`ProvisionerMessage`](../../types/overseer-protocol.md#provisioner-message). Backed candidates come from the [Candidate Backing subsystem](../backing/candidate-backing.md), signed bitfields come from the [Bitfield Distribution subsystem](../availability/bitfield-distribution.md), and disputes come from the [Disputes Subsystem](../disputes/dispute-coordinator.md). Misbehavior reports are currently sent from the [Candidate Backing subsystem](../backing/candidate-backing.md) and contain the following misbehaviors:
+
+1. `Misbehavior::ValidityDoubleVote`
+2. `Misbehavior::MultipleCandidates`
+3. `Misbehavior::UnauthorizedStatement`
+4. `Misbehavior::DoubleSign`
+
+But we choose not to punish these forms of misbehavior for the time being. Risks from misbehavior are sufficiently mitigated at the protocol level via reputation changes. Punitive actions here may become desirable enough to dedicate time to in the future.
 
 At initialization, this subsystem has no outputs.
 

roadmap/implementers-guide/src/types/overseer-protocol.md

@@ -309,7 +309,8 @@ enum CandidateBackingMessage {
   /// The PoV is expected to match the `pov_hash` in the descriptor.
   Second(Hash, CandidateReceipt, PoV),
   /// Note a peer validator's statement about a particular candidate. Disagreements about validity must be escalated
-  /// to a broader check by Misbehavior Arbitration. Agreements are simply tallied until a quorum is reached.
+  /// to a broader check by the Disputes Subsystem, though that escalation is deferred until the approval voting 
+  /// stage to guarantee availability. Agreements are simply tallied until a quorum is reached.
   Statement(Statement),
 }
 ```

runtime/common/src/impls.rs

@@ -133,9 +133,9 @@ mod tests {
 	parameter_types! {
 		pub const BlockHashCount: u64 = 250;
 		pub BlockWeights: limits::BlockWeights = limits::BlockWeights::builder()
-			.base_block(Weight::from_ref_time(10))
+			.base_block(Weight::from_parts(10, 0))
 			.for_class(DispatchClass::all(), |weight| {
-				weight.base_extrinsic = Weight::from_ref_time(100);
+				weight.base_extrinsic = Weight::from_parts(100, 0);
 			})
 			.for_class(DispatchClass::non_mandatory(), |weight| {
 				weight.max_total = Some(Weight::from_parts(1024, u64::MAX));

runtime/common/src/purchase.rs

@@ -196,7 +196,7 @@ pub mod pallet {
 		///
 		/// Origin must match the `ValidityOrigin`.
 		#[pallet::call_index(0)]
-		#[pallet::weight(Weight::from_ref_time(200_000_000) + T::DbWeight::get().reads_writes(4, 1))]
+		#[pallet::weight(Weight::from_parts(200_000_000, 0) + T::DbWeight::get().reads_writes(4, 1))]
 		pub fn create_account(
 			origin: OriginFor<T>,
 			who: T::AccountId,

runtime/kusama/constants/src/weights/block_weights.rs

@@ -52,7 +52,7 @@ parameter_types! {
 	///   95th: 9_411_822
 	///   75th: 9_272_878
 	pub const BlockExecutionWeight: Weight =
-		Weight::from_ref_time(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(9_246_456));
+		Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(9_246_456), 0);
 }
 
 #[cfg(test)]

runtime/kusama/constants/src/weights/extrinsic_weights.rs

@@ -52,7 +52,7 @@ parameter_types! {
 	///   95th: 105_257
 	///   75th: 104_647
 	pub const ExtrinsicBaseWeight: Weight =
-		Weight::from_ref_time(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(104_517));
+		Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(104_517), 0);
 }
 
 #[cfg(test)]

runtime/kusama/src/weights/frame_benchmarking_baseline.rs

@@ -51,53 +51,53 @@ impl<T: frame_system::Config> frame_benchmarking::baseline::WeightInfo for Weigh
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 169 nanoseconds.
-		Weight::from_ref_time(211_793)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(211_793, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 	}
 	/// The range of component `i` is `[0, 1000000]`.
 	fn subtraction(_i: u32, ) -> Weight {
 		// Proof Size summary in bytes:
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 172 nanoseconds.
-		Weight::from_ref_time(210_656)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(210_656, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 	}
 	/// The range of component `i` is `[0, 1000000]`.
 	fn multiplication(_i: u32, ) -> Weight {
 		// Proof Size summary in bytes:
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 173 nanoseconds.
-		Weight::from_ref_time(216_851)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(216_851, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 	}
 	/// The range of component `i` is `[0, 1000000]`.
 	fn division(_i: u32, ) -> Weight {
 		// Proof Size summary in bytes:
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 171 nanoseconds.
-		Weight::from_ref_time(210_747)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(210_747, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 	}
 	fn hashing() -> Weight {
 		// Proof Size summary in bytes:
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 24_801_107 nanoseconds.
-		Weight::from_ref_time(25_036_984_000)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(25_036_984_000, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 	}
 	/// The range of component `i` is `[0, 100]`.
 	fn sr25519_verification(i: u32, ) -> Weight {
 		// Proof Size summary in bytes:
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 197 nanoseconds.
-		Weight::from_ref_time(220_000)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(220_000, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 			// Standard Error: 18_400
-			.saturating_add(Weight::from_ref_time(47_299_555).saturating_mul(i.into()))
+			.saturating_add(Weight::from_parts(47_299_555, 0).saturating_mul(i.into()))
 	}
 }

runtime/kusama/src/weights/frame_election_provider_support.rs

@@ -53,12 +53,12 @@ impl<T: frame_system::Config> frame_election_provider_support::WeightInfo for We
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 6_256_446 nanoseconds.
-		Weight::from_ref_time(6_327_881_000)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(6_327_881_000, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 			// Standard Error: 140_664
-			.saturating_add(Weight::from_ref_time(5_840_511).saturating_mul(v.into()))
+			.saturating_add(Weight::from_parts(5_840_511, 0).saturating_mul(v.into()))
 			// Standard Error: 14_381_047
-			.saturating_add(Weight::from_ref_time(1_543_872_437).saturating_mul(d.into()))
+			.saturating_add(Weight::from_parts(1_543_872_437, 0).saturating_mul(d.into()))
 	}
 	/// The range of component `v` is `[1000, 2000]`.
 	/// The range of component `t` is `[500, 1000]`.
@@ -68,11 +68,11 @@ impl<T: frame_system::Config> frame_election_provider_support::WeightInfo for We
 		//  Measured:  `0`
 		//  Estimated: `0`
 		// Minimum execution time: 4_822_640 nanoseconds.
-		Weight::from_ref_time(4_870_840_000)
-			.saturating_add(Weight::from_proof_size(0))
+		Weight::from_parts(4_870_840_000, 0)
+			.saturating_add(Weight::from_parts(0, 0))
 			// Standard Error: 150_173
-			.saturating_add(Weight::from_ref_time(5_687_544).saturating_mul(v.into()))
+			.saturating_add(Weight::from_parts(5_687_544, 0).saturating_mul(v.into()))
 			// Standard Error: 15_353_175
-			.saturating_add(Weight::from_ref_time(1_784_144_004).saturating_mul(d.into()))
+			.saturating_add(Weight::from_parts(1_784_144_004, 0).saturating_mul(d.into()))
 	}
 }