Merge from develop branch

Cargo.toml

@@ -25,16 +25,15 @@ frame-system-rpc-runtime-api = { path = "pallets/system/rpc/runtime-api" }
 frame-system-benchmarking = { path = "pallets/system/benchmarking" }
 
 # DA Primitives
-# TODO: Once the companion PR 'substrate/upgrade-v1.0.0' is merged, change reference to tag
-avail-core = { version = "0.5", git="https://github.com/availproject/avail-core", branch = "substrate/upgrade-v1.0.0" }
-kate = { version = "0.8", git="https://github.com/availproject/avail-core", branch = "substrate/upgrade-v1.0.0" }
-kate-recovery = { version = "0.9", git="https://github.com/availproject/avail-core", branch = "substrate/upgrade-v1.0.0" }
+avail-core = { version = "0.5", git="https://github.com/availproject/avail-core", branch = "main" }
+kate = { version = "0.8", git="https://github.com/availproject/avail-core", branch = "main" }
+kate-recovery = { version = "0.9", git="https://github.com/availproject/avail-core", branch = "main" }
 
 # Nomad
-nomad-signature = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "substrate/upgrade-v1.0.0" }
-nomad-merkle = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "substrate/upgrade-v1.0.0" }
-nomad-base = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "substrate/upgrade-v1.0.0" }
-nomad-core = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "substrate/upgrade-v1.0.0" }
+nomad-signature = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "main" }
+nomad-merkle = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "main" }
+nomad-base = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "main" }
+nomad-core = { version = "0.1", git="https://github.com/availproject/avail-core", branch = "main" }
 
 # Other stuff
 uint = { git = "https://github.com/paritytech/parity-common.git", tag = "rlp-v0.5.2" }

README.md

@@ -139,6 +139,11 @@ You can sync to the chain using:
 - Warp mode: This is will download the latest state then all the blocks data. It's the fastest and recommended way to have a running node. Use `--sync warp`
 - Fast / Fast Unsafe: This is currently not supported since it does not download data needed for Avail specific computation.
 
+### Unsafe sync
+When importing blocks, their content go through an additional check to make sure that the DA commitments are valid.
+During initial sync, you can chose to ignore this check to increase the sync speed. This command is compatible with any `sync` mode.
+- `--unsafe-da-sync`
+
 
 ## Generate test code coverage report
 

node/src/chain_spec.rs

@@ -239,7 +239,7 @@ pub(crate) mod tests {
 				   network,
 				   transaction_pool,
 				   ..
-			   } = new_full_base(config, |_, _| ())?;
+			   } = new_full_base(config, |_, _| (), false)?;
 			   Ok(sc_service_test::TestNetComponents::new(
 				   task_manager,
 				   client,

node/src/cli.rs

@@ -36,6 +36,10 @@ pub struct Cli {
 	/// telemetry, if telemetry is enabled.
 	#[arg(long)]
 	pub no_hardware_benchmarks: bool,
+
+	/// Disable checking commitment on imported block during sync
+	#[arg(long, conflicts_with_all = &["validator"])]
+	pub unsafe_da_sync: bool,
 }
 
 /// Possible subcommands of the main binary.

node/src/command.rs

@@ -87,7 +87,7 @@ pub fn run() -> Result<()> {
 		None => {
 			let runner = cli.create_runner(&cli.run)?;
 			runner.run_node_until_exit(|config| async move {
-				service::new_full(config, cli.no_hardware_benchmarks)
+				service::new_full(config, cli.no_hardware_benchmarks, cli.unsafe_da_sync)
 					.map_err(sc_cli::Error::Service)
 			})
 		},
@@ -117,7 +117,7 @@ pub fn run() -> Result<()> {
 						unimplemented!();
 						/*
 						// ensure that we keep the task manager alive
-						let partial = new_partial(&config)?;
+						let partial = new_partial(&config, cli.unsafe_da_sync)?;
 						cmd.run(partial.client)
 						*/
 					},
@@ -131,7 +131,7 @@ pub fn run() -> Result<()> {
 						unimplemented!();
 						/*
 						// ensure that we keep the task manager alive
-						let partial = new_partial(&config)?;
+						let partial = new_partial(&config, cli.unsafe_da_sync)?;
 						let db = partial.backend.expose_db();
 						let storage = partial.backend.expose_storage();
 
@@ -142,7 +142,7 @@ pub fn run() -> Result<()> {
 						unimplemented!();
 						/*
 						// ensure that we keep the task manager alive
-						let partial = new_partial(&config)?;
+						let partial = new_partial(&config, cli.unsafe_da_sync)?;
 						let ext_builder = RemarkBuilder::new(partial.client.clone());
 
 						cmd.run(
@@ -158,7 +158,7 @@ pub fn run() -> Result<()> {
 						unimplemented!();
 						/*
 						// ensure that we keep the task manager alive
-						let partial = service::new_partial(&config)?;
+						let partial = service::new_partial(&config, cli.unsafe_da_sync)?;
 						// Register the *Remark* and *TKA* builders.
 						let ext_factory = ExtrinsicFactory(vec![
 							Box::new(RemarkBuilder::new(partial.client.clone())),
@@ -198,7 +198,7 @@ pub fn run() -> Result<()> {
 					task_manager,
 					import_queue,
 					..
-				} = new_partial(&config)?;
+				} = new_partial(&config, cli.unsafe_da_sync)?;
 				Ok((cmd.run(client, import_queue), task_manager))
 			})
 		},
@@ -209,7 +209,7 @@ pub fn run() -> Result<()> {
 					client,
 					task_manager,
 					..
-				} = new_partial(&config)?;
+				} = new_partial(&config, cli.unsafe_da_sync)?;
 				Ok((cmd.run(client, config.database), task_manager))
 			})
 		},
@@ -220,7 +220,7 @@ pub fn run() -> Result<()> {
 					client,
 					task_manager,
 					..
-				} = new_partial(&config)?;
+				} = new_partial(&config, cli.unsafe_da_sync)?;
 				Ok((cmd.run(client, config.chain_spec), task_manager))
 			})
 		},
@@ -232,7 +232,7 @@ pub fn run() -> Result<()> {
 					task_manager,
 					import_queue,
 					..
-				} = new_partial(&config)?;
+				} = new_partial(&config, cli.unsafe_da_sync)?;
 				Ok((cmd.run(client, import_queue), task_manager))
 			})
 		},
@@ -248,7 +248,7 @@ pub fn run() -> Result<()> {
 					task_manager,
 					backend,
 					..
-				} = new_partial(&config)?;
+				} = new_partial(&config, cli.unsafe_da_sync)?;
 				let aux_revert = Box::new(|client: Arc<FullClient>, backend, blocks| {
 					sc_consensus_babe::revert(client.clone(), backend, blocks)?;
 					sc_consensus_grandpa::revert(client, blocks)?;

node/src/da_block_import.rs

@@ -27,13 +27,16 @@ use sp_runtime::traits::Block as BlockT;
 pub struct BlockImport<C, I> {
 	pub client: Arc<C>,
 	pub inner: I,
+	// If true, it skips the DA block import check during sync only.
+	pub unsafe_da_sync: bool,
 }
 
 impl<C, I: Clone> Clone for BlockImport<C, I> {
 	fn clone(&self) -> Self {
 		Self {
 			client: self.client.clone(),
 			inner: self.inner.clone(),
+			unsafe_da_sync: self.unsafe_da_sync,
 		}
 	}
 }
@@ -57,7 +60,18 @@ where
 		block: BlockImportParams<B, Self::Transaction>,
 	) -> Result<ImportResult, Self::Error> {
 		// We only want to check for blocks that are not from "Own"
-		if !matches!(block.origin, BlockOrigin::Own) {
+		let is_own = matches!(block.origin, BlockOrigin::Own);
+
+		// We skip checks if we're syncing and unsafe_da_sync is true
+		let is_sync = matches!(
+			block.origin,
+			BlockOrigin::NetworkInitialSync | BlockOrigin::File
+		);
+		let skip_sync = self.unsafe_da_sync && is_sync;
+
+		let should_verify = !is_own && !skip_sync;
+		println!("Should verify: {should_verify}");
+		if should_verify {
 			let no_extrinsics = vec![];
 			let extrinsics = block.body.as_ref().unwrap_or(&no_extrinsics);
 			let best_hash = self.client.info().best_hash;

node/src/service.rs

@@ -161,6 +161,7 @@ pub fn create_extrinsic(
 #[allow(clippy::type_complexity)]
 pub fn new_partial(
 	config: &Configuration,
+	unsafe_da_sync: bool,
 ) -> Result<
 	sc_service::PartialComponents<
 		FullClient,
@@ -235,7 +236,8 @@ pub fn new_partial(
 		grandpa_block_import,
 		client.clone(),
 	)?;
-	let da_block_import = BlockImport::new(client.clone(), block_import);
+
+	let da_block_import = BlockImport::new(client.clone(), block_import, unsafe_da_sync);
 
 	let slot_duration = babe_link.config().slot_duration();
 	let (import_queue, babe_worker_handle) =
@@ -343,6 +345,7 @@ pub fn new_full_base(
 	config: Configuration,
 	disable_hardware_benchmarks: bool,
 	with_startup_data: impl FnOnce(&BlockImport, &sc_consensus_babe::BabeLink<Block>),
+	unsafe_da_sync: bool,
 ) -> Result<NewFullBase, ServiceError> {
 	let hwbench = if !disable_hardware_benchmarks {
 		config.database.path().map(|database_path| {
@@ -362,7 +365,7 @@ pub fn new_full_base(
 		select_chain,
 		transaction_pool,
 		other: (rpc_builder, import_setup, rpc_setup, mut telemetry),
-	} = new_partial(&config)?;
+	} = new_partial(&config, unsafe_da_sync)?;
 
 	let shared_voter_state = rpc_setup;
 	let auth_disc_publish_non_global_ips = config.network.allow_non_globals_in_dht;
@@ -591,9 +594,15 @@ pub fn new_full_base(
 pub fn new_full(
 	config: Configuration,
 	disable_hardware_benchmarks: bool,
+	unsafe_da_sync: bool,
 ) -> Result<TaskManager, ServiceError> {
-	new_full_base(config, disable_hardware_benchmarks, |_, _| ())
-		.map(|NewFullBase { task_manager, .. }| task_manager)
+	new_full_base(
+		config,
+		disable_hardware_benchmarks,
+		|_, _| (),
+		unsafe_da_sync,
+	)
+	.map(|NewFullBase { task_manager, .. }| task_manager)
 }
 
 fn extend_metrics(prometheus: &Registry) -> Result<(), PrometheusError> {
@@ -676,6 +685,7 @@ mod tests {
 					 babe_link: &sc_consensus_babe::BabeLink<Block>| {
 						setup_handles = Some((block_import.clone(), babe_link.clone()));
 					},
+					false,
 				)?;
 
 				let node = sc_service_test::TestNetComponents::new(
@@ -869,7 +879,7 @@ mod tests {
 					network,
 					transaction_pool,
 					..
-				} = new_full_base(config, |_, _| ())?;
+				} = new_full_base(config, |_, _| (), false)?;
 				Ok(sc_service_test::TestNetComponents::new(
 					task_manager,
 					client,

pallets/mandate/src/mock.rs

@@ -103,6 +103,11 @@ where
 			r => Err(OuterOrigin::from(r)),
 		})
 	}
+
+	#[cfg(feature = "runtime-benchmarks")]
+	fn try_successful_origin() -> Result<OuterOrigin, ()> {
+		unimplemented!()
+	}
 }
 
 /// Create new externalities for `Mandate` module tests.

pallets/system/Cargo.toml

@@ -39,6 +39,8 @@ sp-weights = { version = "20.0.0", default-features = false, features = ["serde"
 sp-runtime-interface = { version = "17", default-features = false }
 binary-merkle-tree = { version = "4.0.0-dev", default-features = false }
 
+once_cell = { version = "1.18", optional = true }
+
 [dev-dependencies]
 hex-literal = "0.3.1"
 test-case = "1.2.3"
@@ -64,6 +66,7 @@ std = [
 	"sp-version/std",
 	"sp-weights/std",
 	"binary-merkle-tree/std",
+	"once_cell"
 ]
 runtime-benchmarks = [
 	"frame-support/runtime-benchmarks",