Offchain Phragmén BREAKING. (#4517)

* Initial skeleton for offchain phragmen

* Basic compact encoding decoding for results

* add compact files

* Bring back Self::ensure_storage_upgraded();

* Make staking use compact stuff.

* First seemingly working version of reduce, full of todos

* Everything phragmen related works again.

* Signing made easier, still issues.

* Signing from offchain compile fine 😎

* make compact work with staked asssignment

* Evaluation basics are in place.

* Move reduce into crate. Document stuff

* move reduce into no_std

* Add files

* Remove other std deps. Runtime compiles

* Seemingly it is al stable; cycle implemented but not integrated.

* Add fuzzing code.

* Cleanup reduce a bit more.

* a metric ton of tests for staking; wip 🔨

* Implement a lot more of the tests.

* wip getting the unsigned stuff to work

* A bit gleanup for unsigned debug

* Clean and finalize compact code.

* Document reduce.

* Still problems with signing

* We officaly duct taped the transaction submission stuff. 🤓

* Deadlock with keys again

* Runtime builds

* Unsigned test works 🙌

* Some cleanups

* Make all the tests compile and stuff

* Minor cleanup

* fix more merge stuff

* Most tests work again.

* a very nasty bug in reduce

* Fix all integrations

* Fix more todos

* Revamp everything and everything

* Remove bogus test

* Some review grumbles.

* Some fixes

* Fix doc test

* loop for submission

* Fix cli, keyring etc.

* some cleanup

* Fix staking tests again

* fix per-things; bring patches from benchmarking

* better score prediction

* Add fuzzer, more patches.

* Some fixes

* More docs

* Remove unused generics

* Remove max-nominator footgun

* Better fuzzer

* Disable it ❌

* Bump.

* Another round of self-review

* Refactor a lot

* More major fixes in perThing

* Add new fuzz file

* Update lock

* fix fuzzing code.

* Fix nominator retain test

* Add slashing check

* Update frame/staking/src/tests.rs

Co-Authored-By: Joshy Orndorff <[email protected]>

* Some formatting nits

* Review comments.

* Fix cargo file

* Almost all tests work again

* Update frame/staking/src/tests.rs

Co-Authored-By: thiolliere <[email protected]>

* Fix review comments

* More review stuff

* Some nits

* Fix new staking / session / babe relation

* Update primitives/phragmen/src/lib.rs

Co-Authored-By: thiolliere <[email protected]>

* Update primitives/phragmen/src/lib.rs

Co-Authored-By: thiolliere <[email protected]>

* Update primitives/phragmen/compact/src/lib.rs

Co-Authored-By: thiolliere <[email protected]>

* Some doc updates to slashing

* Fix derive

* Remove imports

* Remove unimplemented tests

* nits

* Remove dbg

* Better fuzzing params

* Remove unused pref map

* Deferred Slashing/Offence for offchain Phragmen  (#5151)

* Some boilerplate

* Add test

* One more test

* Review comments

* Fix build

* review comments

* fix more

* fix build

* Some cleanups and self-reviews

* More minor self reviews

* Final nits

* Some merge fixes.

* opt comment

* Fix build

* Fix build again.

* Update frame/staking/fuzz/fuzz_targets/submit_solution.rs

Co-Authored-By: Gavin Wood <[email protected]>

* Update frame/staking/src/slashing.rs

Co-Authored-By: Gavin Wood <[email protected]>

* Update frame/staking/src/offchain_election.rs

Co-Authored-By: Gavin Wood <[email protected]>

* Fix review comments

* fix test

* === 🔑 Revamp without staking key.

* final round of changes.

* Fix cargo-deny

* Update frame/staking/src/lib.rs

Co-Authored-By: Gavin Wood <[email protected]>

Co-authored-by: Joshy Orndorff <[email protected]>
Co-authored-by: thiolliere <[email protected]>
Co-authored-by: Gavin Wood <[email protected]>

Cargo.toml

@@ -130,6 +130,7 @@ members = [
 	"primitives/offchain",
 	"primitives/panic-handler",
 	"primitives/phragmen",
+	"primitives/phragmen/compact",
 	"primitives/rpc",
 	"primitives/runtime-interface",
 	"primitives/runtime-interface/proc-macro",

bin/node/cli/Cargo.toml

@@ -82,6 +82,7 @@ pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../../../frame
 frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/support" }
 pallet-im-online = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/im-online" }
 pallet-authority-discovery = { version = "2.0.0-alpha.5",  path = "../../../frame/authority-discovery" }
+pallet-staking = { version = "2.0.0-alpha.5",  path = "../../../frame/staking" }
 
 # node-specific dependencies
 node-runtime = { version = "2.0.0-alpha.5", path = "../runtime" }

bin/node/cli/src/chain_spec.rs

@@ -202,7 +202,14 @@ pub fn get_authority_keys_from_seed(seed: &str) -> (
 
 /// Helper function to create GenesisConfig for testing
 pub fn testnet_genesis(
-	initial_authorities: Vec<(AccountId, AccountId, GrandpaId, BabeId, ImOnlineId, AuthorityDiscoveryId)>,
+	initial_authorities: Vec<(
+		AccountId,
+		AccountId,
+		GrandpaId,
+		BabeId,
+		ImOnlineId,
+		AuthorityDiscoveryId,
+	)>,
 	root_key: AccountId,
 	endowed_accounts: Option<Vec<AccountId>>,
 	enable_println: bool,
@@ -244,7 +251,12 @@ pub fn testnet_genesis(
 		}),
 		pallet_session: Some(SessionConfig {
 			keys: initial_authorities.iter().map(|x| {
-				(x.0.clone(), x.0.clone(), session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone()))
+				(x.0.clone(), x.0.clone(), session_keys(
+					x.2.clone(),
+					x.3.clone(),
+					x.4.clone(),
+					x.5.clone(),
+				))
 			}).collect::<Vec<_>>(),
 		}),
 		pallet_staking: Some(StakingConfig {

bin/node/cli/src/factory_impl.rs

@@ -58,6 +58,7 @@ impl<Number> FactoryState<Number> {
 			frame_system::CheckWeight::new(),
 			pallet_transaction_payment::ChargeTransactionPayment::from(0),
 			Default::default(),
+			Default::default(),
 		)
 	}
 }
@@ -122,7 +123,7 @@ impl RuntimeAdapter for FactoryState<Number> {
 					(*amount).into()
 				)
 			)
-		}, key, (version, genesis_hash.clone(), prior_block_hash.clone(), (), (), (), ()))
+		}, key, (version, genesis_hash.clone(), prior_block_hash.clone(), (), (), (), (), ()))
 	}
 
 	fn inherent_extrinsics(&self) -> InherentData {

bin/node/cli/src/service.rs

@@ -624,11 +624,12 @@ mod tests {
 					check_weight,
 					payment,
 					Default::default(),
+					Default::default(),
 				);
 				let raw_payload = SignedPayload::from_raw(
 					function,
 					extra,
-					(version, genesis_hash, genesis_hash, (), (), (), ())
+					(version, genesis_hash, genesis_hash, (), (), (), (), ())
 				);
 				let signature = raw_payload.using_encoded(|payload|	{
 					signer.sign(payload)

bin/node/executor/tests/submit_transaction.rs

@@ -15,7 +15,7 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use node_runtime::{
-	Call, Executive, Indices, Runtime, SubmitTransaction, UncheckedExtrinsic,
+	Call, Executive, Indices, Runtime, TransactionSubmitterOf, UncheckedExtrinsic,
 };
 use sp_application_crypto::AppKey;
 use sp_core::testing::KeyStore;
@@ -31,6 +31,8 @@ use codec::Decode;
 pub mod common;
 use self::common::*;
 
+type SubmitTransaction = TransactionSubmitterOf<pallet_im_online::sr25519::AuthorityId>;
+
 #[test]
 fn should_submit_unsigned_transaction() {
 	let mut t = new_test_ext(COMPACT_CODE, false);

bin/node/runtime/src/lib.rs

@@ -59,7 +59,7 @@ pub use pallet_timestamp::Call as TimestampCall;
 pub use pallet_balances::Call as BalancesCall;
 pub use pallet_contracts::Gas;
 pub use frame_support::StorageValue;
-pub use pallet_staking::StakerStatus;
+pub use pallet_staking::{StakerStatus, LockStakingStatus};
 
 /// Implementations of some helper traits passed into runtime modules as associated types.
 pub mod impls;
@@ -73,6 +73,52 @@ use constants::{time::*, currency::*};
 #[cfg(feature = "std")]
 include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
 
+/// A transaction submitter with the given key type.
+pub type TransactionSubmitterOf<KeyType> = TransactionSubmitter<KeyType, Runtime, UncheckedExtrinsic>;
+
+/// Submits transaction with the node's public and signature type. Adheres to the signed extension
+/// format of the chain.
+impl frame_system::offchain::CreateTransaction<Runtime, UncheckedExtrinsic> for Runtime {
+	type Public = <Signature as traits::Verify>::Signer;
+	type Signature = Signature;
+
+	fn create_transaction<TSigner: frame_system::offchain::Signer<Self::Public, Self::Signature>>(
+		call: Call,
+		public: Self::Public,
+		account: AccountId,
+		index: Index,
+	) -> Option<(Call, <UncheckedExtrinsic as traits::Extrinsic>::SignaturePayload)> {
+		// take the biggest period possible.
+		let period = BlockHashCount::get()
+			.checked_next_power_of_two()
+			.map(|c| c / 2)
+			.unwrap_or(2) as u64;
+		let current_block = System::block_number()
+			.saturated_into::<u64>()
+			// The `System::block_number` is initialized with `n+1`,
+			// so the actual block number is `n`.
+			.saturating_sub(1);
+		let tip = 0;
+		let extra: SignedExtra = (
+			frame_system::CheckVersion::<Runtime>::new(),
+			frame_system::CheckGenesis::<Runtime>::new(),
+			frame_system::CheckEra::<Runtime>::from(generic::Era::mortal(period, current_block)),
+			frame_system::CheckNonce::<Runtime>::from(index),
+			frame_system::CheckWeight::<Runtime>::new(),
+			pallet_transaction_payment::ChargeTransactionPayment::<Runtime>::from(tip),
+			Default::default(),
+			Default::default(),
+		);
+		let raw_payload = SignedPayload::new(call, extra).map_err(|e| {
+			debug::warn!("Unable to create signed payload: {:?}", e);
+		}).ok()?;
+		let signature = TSigner::sign(public, &raw_payload)?;
+		let address = Indices::unlookup(account);
+		let (call, extra, _) = raw_payload.deconstruct();
+		Some((call, (address, signature, extra)))
+	}
+}
+
 /// Runtime version.
 pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("node"),
@@ -255,6 +301,7 @@ impl pallet_session::Trait for Runtime {
 	type SessionHandler = <SessionKeys as OpaqueKeys>::KeyTypeIdProviders;
 	type Keys = SessionKeys;
 	type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
+	type NextSessionRotation = Babe;
 }
 
 impl pallet_session::historical::Trait for Runtime {
@@ -278,6 +325,7 @@ parameter_types! {
 	pub const BondingDuration: pallet_staking::EraIndex = 24 * 28;
 	pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration.
 	pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE;
+	pub const ElectionLookahead: BlockNumber = 25; // 10 minutes per session => 100 block.
 	pub const MaxNominatorRewardedPerValidator: u32 = 64;
 }
 
@@ -296,6 +344,10 @@ impl pallet_staking::Trait for Runtime {
 	type SlashCancelOrigin = pallet_collective::EnsureProportionAtLeast<_3, _4, AccountId, CouncilCollective>;
 	type SessionInterface = Self;
 	type RewardCurve = RewardCurve;
+	type NextNewSession = Session;
+	type ElectionLookahead = ElectionLookahead;
+	type Call = Call;
+	type SubmitTransaction = TransactionSubmitterOf<()>;
 	type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator;
 }
 
@@ -470,9 +522,6 @@ impl pallet_sudo::Trait for Runtime {
 	type Call = Call;
 }
 
-/// A runtime transaction submitter.
-pub type SubmitTransaction = TransactionSubmitter<ImOnlineId, Runtime, UncheckedExtrinsic>;
-
 parameter_types! {
 	pub const SessionDuration: BlockNumber = EPOCH_DURATION_IN_SLOTS as _;
 }
@@ -481,7 +530,7 @@ impl pallet_im_online::Trait for Runtime {
 	type AuthorityId = ImOnlineId;
 	type Event = Event;
 	type Call = Call;
-	type SubmitTransaction = SubmitTransaction;
+	type SubmitTransaction = TransactionSubmitterOf<Self::AuthorityId>;
 	type SessionDuration = SessionDuration;
 	type ReportUnresponsiveness = Offences;
 }
@@ -530,46 +579,6 @@ impl pallet_identity::Trait for Runtime {
 	type RegistrarOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>;
 }
 
-impl frame_system::offchain::CreateTransaction<Runtime, UncheckedExtrinsic> for Runtime {
-	type Public = <Signature as traits::Verify>::Signer;
-	type Signature = Signature;
-
-	fn create_transaction<TSigner: frame_system::offchain::Signer<Self::Public, Self::Signature>>(
-		call: Call,
-		public: Self::Public,
-		account: AccountId,
-		index: Index,
-	) -> Option<(Call, <UncheckedExtrinsic as traits::Extrinsic>::SignaturePayload)> {
-		// take the biggest period possible.
-		let period = BlockHashCount::get()
-			.checked_next_power_of_two()
-			.map(|c| c / 2)
-			.unwrap_or(2) as u64;
-		let current_block = System::block_number()
-			.saturated_into::<u64>()
-			// The `System::block_number` is initialized with `n+1`,
-			// so the actual block number is `n`.
-			.saturating_sub(1);
-		let tip = 0;
-		let extra: SignedExtra = (
-			frame_system::CheckVersion::<Runtime>::new(),
-			frame_system::CheckGenesis::<Runtime>::new(),
-			frame_system::CheckEra::<Runtime>::from(generic::Era::mortal(period, current_block)),
-			frame_system::CheckNonce::<Runtime>::from(index),
-			frame_system::CheckWeight::<Runtime>::new(),
-			pallet_transaction_payment::ChargeTransactionPayment::<Runtime>::from(tip),
-			Default::default(),
-		);
-		let raw_payload = SignedPayload::new(call, extra).map_err(|e| {
-			debug::warn!("Unable to create signed payload: {:?}", e);
-		}).ok()?;
-		let signature = TSigner::sign(public, &raw_payload)?;
-		let address = Indices::unlookup(account);
-		let (call, extra, _) = raw_payload.deconstruct();
-		Some((call, (address, signature, extra)))
-	}
-}
-
 parameter_types! {
 	pub const ConfigDepositBase: Balance = 5 * DOLLARS;
 	pub const FriendDepositFactor: Balance = 50 * CENTS;
@@ -638,7 +647,7 @@ construct_runtime!(
 		Indices: pallet_indices::{Module, Call, Storage, Config<T>, Event<T>},
 		Balances: pallet_balances::{Module, Call, Storage, Config<T>, Event<T>},
 		TransactionPayment: pallet_transaction_payment::{Module, Storage},
-		Staking: pallet_staking::{Module, Call, Config<T>, Storage, Event<T>},
+		Staking: pallet_staking::{Module, Call, Config<T>, Storage, Event<T>, ValidateUnsigned},
 		Session: pallet_session::{Module, Call, Storage, Event, Config<T>},
 		Democracy: pallet_democracy::{Module, Call, Storage, Config, Event<T>},
 		Council: pallet_collective::<Instance1>::{Module, Call, Storage, Origin<T>, Event<T>, Config<T>},
@@ -680,6 +689,7 @@ pub type SignedExtra = (
 	frame_system::CheckWeight<Runtime>,
 	pallet_transaction_payment::ChargeTransactionPayment<Runtime>,
 	pallet_contracts::CheckBlockGasLimit<Runtime>,
+	pallet_staking::LockStakingStatus<Runtime>,
 );
 /// Unchecked extrinsic type as expected by this runtime.
 pub type UncheckedExtrinsic = generic::UncheckedExtrinsic<Address, Call, Signature, SignedExtra>;
@@ -965,7 +975,7 @@ mod tests {
 			>,
 		{}
 
-		is_submit_signed_transaction::<SubmitTransaction>();
-		is_sign_and_submit_transaction::<SubmitTransaction>();
+		is_submit_signed_transaction::<TransactionSubmitterOf<ImOnlineId>>();
+		is_sign_and_submit_transaction::<TransactionSubmitterOf<ImOnlineId>>();
 	}
 }

bin/node/testing/src/keyring.rs

@@ -75,6 +75,7 @@ pub fn signed_extra(nonce: Index, extra_fee: Balance) -> SignedExtra {
 		frame_system::CheckWeight::new(),
 		pallet_transaction_payment::ChargeTransactionPayment::from(extra_fee),
 		Default::default(),
+		Default::default(),
 	)
 }
 

bin/utils/subkey/src/main.rs

@@ -683,6 +683,7 @@ fn create_extrinsic<C: Crypto>(
 			frame_system::CheckWeight::<Runtime>::new(),
 			pallet_transaction_payment::ChargeTransactionPayment::<Runtime>::from(f),
 			Default::default(),
+			Default::default(),
 		)
 	};
 	let raw_payload = SignedPayload::from_raw(
@@ -696,6 +697,7 @@ fn create_extrinsic<C: Crypto>(
 			(),
 			(),
 			(),
+			(),
 		),
 	);
 	let signature = raw_payload.using_encoded(|payload| signer.sign(payload)).into_runtime();

frame/authority-discovery/src/lib.rs

@@ -121,6 +121,7 @@ mod tests {
 		type ValidatorId = AuthorityId;
 		type ValidatorIdOf = ConvertInto;
 		type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
+		type NextSessionRotation = pallet_session::PeriodicSessions<Period, Offset>;
 	}
 
 	impl pallet_session::historical::Trait for Test {

frame/babe/Cargo.toml

@@ -9,7 +9,6 @@ repository = "https://github.com/paritytech/substrate/"
 description = "Consensus extension module for BABE consensus. Collects on-chain randomness from VRF outputs and manages epoch transitions."
 
 [dependencies]
-hex-literal = "0.2.1"
 codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] }
 serde = { version = "1.0.101", optional = true }
 sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" }
@@ -26,11 +25,7 @@ sp-consensus-vrf = { version = "0.8.0-alpha.5", default-features = false, path =
 sp-io = { path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"}
 
 [dev-dependencies]
-lazy_static = "1.4.0"
-parking_lot = "0.10.0"
-sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/version" }
 sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" }
-substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" }
 
 [features]
 default = ["std"]