elastic scaling RFC 103 end-to-end test

.gitlab/pipeline/zombienet/polkadot.yml

@@ -190,6 +190,36 @@ zombienet-polkadot-elastic-scaling-0002-elastic-scaling-doesnt-break-parachains:
       --local-dir="${LOCAL_DIR}/elastic_scaling"
       --test="0002-elastic-scaling-doesnt-break-parachains.zndsl"
 
+zombienet-polkadot-elastic-scaling-0003-slot-based-3cores:
+  extends:
+    - .zombienet-polkadot-common
+  variables:
+    FORCED_INFRA_INSTANCE: "spot-iops"
+    # override COL_IMAGE to use test-parachain
+    COL_IMAGE: "docker.io/paritypr/test-parachain:${PIPELINE_IMAGE_TAG}"
+  before_script:
+    - !reference [ .zombienet-polkadot-common, before_script ]
+    - cp --remove-destination ${LOCAL_DIR}/assign-core.js ${LOCAL_DIR}/elastic_scaling
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}/elastic_scaling"
+      --test="0003-slot-based-3cores.zndsl"
+
+zombienet-polkadot-elastic-scaling-0004-mixed-receipt-versions:
+  extends:
+    - .zombienet-polkadot-common
+  variables:
+    FORCED_INFRA_INSTANCE: "spot-iops"
+    # override COL_IMAGE to use test-parachain
+    COL_IMAGE: "docker.io/paritypr/test-parachain:${PIPELINE_IMAGE_TAG}"
+  before_script:
+    - !reference [ .zombienet-polkadot-common, before_script ]
+    - cp --remove-destination ${LOCAL_DIR}/assign-core.js ${LOCAL_DIR}/elastic_scaling
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}/elastic_scaling"
+      --test="0004-mixed-receipt-versions.zndsl"      
+
 .zombienet-polkadot-functional-0012-spam-statement-distribution-requests:
   extends:
     - .zombienet-polkadot-common

cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml

@@ -49,7 +49,7 @@ polkadot-runtime-common = { workspace = true }
 # Cumulus
 cumulus-pallet-aura-ext = { workspace = true }
 pallet-message-queue = { workspace = true }
-cumulus-pallet-parachain-system = { workspace = true, features = ["experimental-ump-signals"] }
+cumulus-pallet-parachain-system = { workspace = true }
 cumulus-pallet-xcm = { workspace = true }
 cumulus-pallet-xcmp-queue = { workspace = true }
 cumulus-ping = { workspace = true }

cumulus/test/runtime/Cargo.toml

@@ -96,3 +96,4 @@ std = [
 ]
 increment-spec-version = []
 elastic-scaling = []
+experimental-ump-signals = ["cumulus-pallet-parachain-system/experimental-ump-signals"]

cumulus/test/runtime/build.rs

@@ -29,6 +29,14 @@ fn main() {
 		.with_current_project()
 		.enable_feature("elastic-scaling")
 		.import_memory()
+		.set_file_name("wasm_binary_elastic_scaling_mvp.rs")
+		.build();
+
+	WasmBuilder::new()
+		.with_current_project()
+		.enable_feature("elastic-scaling")
+		.enable_feature("experimental-ump-signals")
+		.import_memory()
 		.set_file_name("wasm_binary_elastic_scaling.rs")
 		.build();
 }

cumulus/test/runtime/src/lib.rs

@@ -27,6 +27,11 @@ pub mod wasm_spec_version_incremented {
 	include!(concat!(env!("OUT_DIR"), "/wasm_binary_spec_version_incremented.rs"));
 }
 
+pub mod elastic_scaling_mvp {
+	#[cfg(feature = "std")]
+	include!(concat!(env!("OUT_DIR"), "/wasm_binary_elastic_scaling_mvp.rs"));
+}
+
 pub mod elastic_scaling {
 	#[cfg(feature = "std")]
 	include!(concat!(env!("OUT_DIR"), "/wasm_binary_elastic_scaling.rs"));

cumulus/test/service/src/chain_spec.rs

@@ -116,3 +116,13 @@ pub fn get_elastic_scaling_chain_spec(id: Option<ParaId>) -> ChainSpec {
 			.expect("WASM binary was not built, please build it!"),
 	)
 }
+
+/// Get the chain spec for a specific parachain ID.
+pub fn get_elastic_scaling_mvp_chain_spec(id: Option<ParaId>) -> ChainSpec {
+	get_chain_spec_with_extra_endowed(
+		id,
+		Default::default(),
+		cumulus_test_runtime::elastic_scaling_mvp::WASM_BINARY
+			.expect("WASM binary was not built, please build it!"),
+	)
+}

cumulus/test/service/src/cli.rs

@@ -262,10 +262,16 @@ impl SubstrateCli for TestCollatorCli {
 				tracing::info!("Using default test service chain spec.");
 				Box::new(cumulus_test_service::get_chain_spec(Some(ParaId::from(2000)))) as Box<_>
 			},
+			"elastic-scaling-mvp" => {
+				tracing::info!("Using elastic-scaling mvp chain spec.");
+				Box::new(cumulus_test_service::get_elastic_scaling_mvp_chain_spec(Some(
+					ParaId::from(2100),
+				))) as Box<_>
+			},
 			"elastic-scaling" => {
 				tracing::info!("Using elastic-scaling chain spec.");
 				Box::new(cumulus_test_service::get_elastic_scaling_chain_spec(Some(ParaId::from(
-					2100,
+					2200,
 				)))) as Box<_>
 			},
 			path => {

polkadot/runtime/rococo/src/genesis_config_presets.rs

@@ -129,7 +129,9 @@ fn default_parachains_host_configuration(
 			allowed_ancestry_len: 2,
 		},
 		node_features: bitvec::vec::BitVec::from_element(
-			1u8 << (FeatureIndex::ElasticScalingMVP as usize),
+			1u8 << (FeatureIndex::ElasticScalingMVP as usize) |
+				1u8 << (FeatureIndex::EnableAssignmentsV2 as usize) |
+				1u8 << (FeatureIndex::CandidateReceiptV2 as usize),
 		),
 		scheduler_params: SchedulerParams {
 			lookahead: 2,

polkadot/runtime/westend/src/genesis_config_presets.rs

@@ -133,7 +133,8 @@ fn default_parachains_host_configuration(
 		},
 		node_features: bitvec::vec::BitVec::from_element(
 			1u8 << (FeatureIndex::ElasticScalingMVP as usize) |
-				1u8 << (FeatureIndex::EnableAssignmentsV2 as usize),
+				1u8 << (FeatureIndex::EnableAssignmentsV2 as usize) |
+				1u8 << (FeatureIndex::CandidateReceiptV2 as usize),
 		),
 		scheduler_params: SchedulerParams {
 			lookahead: 2,

polkadot/zombienet_tests/elastic_scaling/0003-slot-based-3cores.toml

@@ -0,0 +1,48 @@
+[settings]
+timeout = 1000
+
+[relaychain.genesis.runtimeGenesis.patch.configuration.config.async_backing_params]
+  max_candidate_depth = 6
+  allowed_ancestry_len = 2
+
+[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params]
+  max_validators_per_core = 1
+  num_cores = 4
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+chain = "rococo-local"
+default_command = "polkadot"
+
+  [relaychain.default_resources]
+  limits = { memory = "4G", cpu = "3" }
+  requests = { memory = "4G", cpu = "3" }
+
+  [[relaychain.node_groups]]
+  name = "elastic-validator"
+  count = 6
+  args = [ "-lparachain=debug,parachain::candidate-backing=trace,parachain::provisioner=trace,parachain::prospective-parachains=trace,runtime=debug"]
+
+# First collator uses elastic scaling MVP.
+[[parachains]]
+id = 2100
+chain = "elastic-scaling-mvp"
+add_to_genesis = true
+
+  [[parachains.collators]]
+  name = "collator-elastic-mvp"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["-laura=trace,runtime=info,cumulus-consensus=trace,consensus::common=trace,parachain::collation-generation=trace,parachain::collator-protocol=trace,parachain=debug", "--force-authoring", "--experimental-use-slot-based"]
+
+# Second collator uses UMP commitments (RFC 103).
+[[parachains]]
+id = 2200
+chain = "elastic-scaling"
+add_to_genesis = true
+
+  [[parachains.collators]]
+  name = "collator-elastic"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["-laura=trace,runtime=info,cumulus-consensus=trace,consensus::common=trace,parachain::collation-generation=trace,parachain::collator-protocol=trace,parachain=debug", "--force-authoring", "--experimental-use-slot-based"]

polkadot/zombienet_tests/elastic_scaling/0003-slot-based-3cores.zndsl

@@ -0,0 +1,19 @@
+Description: Test with slot-based collator using 3 cores and async backing
+Network: ./0003-slot-based-3cores.toml
+Creds: config
+
+elastic-validator: is up
+collator-elastic: is up
+collator-elastic-mvp: is up
+
+# Register 2 extra cores to each parachain.
+elastic-validator-0: js-script ./assign-core.js with "0,2100,57600" return is 0 within 600 seconds
+elastic-validator-1: js-script ./assign-core.js with "1,2100,57600" return is 0 within 600 seconds
+elastic-validator-2: js-script ./assign-core.js with "2,2200,57600" return is 0 within 600 seconds
+elastic-validator-3: js-script ./assign-core.js with "3,2200,57600" return is 0 within 600 seconds
+
+# Wait for 20 relay chain blocks 
+elastic-validator-0: reports substrate_block_height{status="best"} is at least 20 within 140 seconds
+
+elastic-validator-0: parachain 2100 block height is at least 20 within 10 seconds
+elastic-validator-0: parachain 2200 block height is at least 20 within 10 seconds

polkadot/zombienet_tests/elastic_scaling/0004-mixed-receipt-versions.toml

@@ -0,0 +1,42 @@
+[settings]
+timeout = 1000
+
+[relaychain.genesis.runtimeGenesis.patch.configuration.config.async_backing_params]
+  max_candidate_depth = 6
+  allowed_ancestry_len = 2
+
+[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params]
+  max_validators_per_core = 1
+  num_cores = 2
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+chain = "rococo-local"
+default_command = "polkadot"
+
+  [relaychain.default_resources]
+  limits = { memory = "4G", cpu = "3" }
+  requests = { memory = "4G", cpu = "3" }
+
+  [[relaychain.node_groups]]
+  name = "elastic-validator"
+  count = 3
+  args = [ "-lparachain=debug,parachain::candidate-backing=trace,parachain::provisioner=trace,parachain::prospective-parachains=trace,runtime=debug"]
+
+[[parachains]]
+id = 2200
+chain = "elastic-scaling"
+add_to_genesis = true
+
+  [[parachains.collators]]
+  name = "collator-elastic"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["-laura=trace,runtime=info,cumulus-consensus=trace,consensus::common=trace,parachain::collation-generation=trace,parachain::collator-protocol=trace,parachain=debug", "--force-authoring", "--experimental-use-slot-based"]
+
+  [[parachains.collators]]
+  name = "old-collator-elastic"
+  # Use an old image here, this is roughly the commit where the stable2407 release branched off.
+  image = "{{COL_IMAGE}}:master-b862b181"
+  command = "test-parachain"
+  args = ["-laura=trace,runtime=info,cumulus-consensus=trace,consensus::common=trace,parachain::collation-generation=trace,parachain::collator-protocol=trace,parachain=debug", "--force-authoring", "--experimental-use-slot-based"]

polkadot/zombienet_tests/elastic_scaling/0004-mixed-receipt-versions.zndsl

@@ -0,0 +1,17 @@
+Description: Test 3-core elastic scaling with two slot-based collators using both receipt versions
+Network: ./0004-mixed-receipt-versions.toml
+Creds: config
+
+elastic-validator: is up
+collator-elastic: is up
+old-collator-elastic: is up
+
+# Register 2 extra cores to the parachain.
+elastic-validator-0: js-script ./assign-core.js with "0,2200,57600" return is 0 within 600 seconds
+elastic-validator-1: js-script ./assign-core.js with "1,2200,57600" return is 0 within 600 seconds
+
+# Wait for 20 relay chain blocks 
+elastic-validator-0: reports substrate_block_height{status="best"} is at least 20 within 140 seconds
+
+# We won't get full throughput as some candidates would be built for the same block and therefore dropped in the runtime.
+elastic-validator-0: parachain 2200 block height is at least 16 within 10 seconds