diff --git a/.github/scripts/cmd/cmd.py b/.github/scripts/cmd/cmd.py index b421ab753f78..01f36ea375c5 100755 --- a/.github/scripts/cmd/cmd.py +++ b/.github/scripts/cmd/cmd.py @@ -104,7 +104,7 @@ def main(): print(f'-- listing pallets for benchmark for {runtime["name"]}') wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm" output = os.popen( - f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file}").read() + f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file} {runtime['bench_flags']}").read() raw_pallets = output.strip().split('\n') all_pallets = set() @@ -182,7 +182,8 @@ def main(): f"--repeat=20 " \ f"--heap-pages=4096 " \ f"{f'--template={template} ' if template else ''}" \ - f"--no-storage-info --no-min-squares --no-median-slopes" + f"--no-storage-info --no-min-squares --no-median-slopes " \ + f"{config['bench_flags']}" print(f'-- Running: {cmd} \n') status = os.system(cmd) if status != 0 and not args.continue_on_fail: diff --git a/.github/scripts/cmd/test_cmd.py b/.github/scripts/cmd/test_cmd.py index f4e70903347f..0316c7ff1bb4 100644 --- a/.github/scripts/cmd/test_cmd.py +++ b/.github/scripts/cmd/test_cmd.py @@ -7,13 +7,45 @@ # Mock data for runtimes-matrix.json mock_runtimes_matrix = [ - {"name": "dev", "package": "kitchensink-runtime", "path": "substrate/frame", "header": "substrate/HEADER-APACHE2", "template": "substrate/.maintain/frame-weight-template.hbs", "bench_features": "runtime-benchmarks,riscv"}, - {"name": "westend", "package": "westend-runtime", "path": "polkadot/runtime/westend", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", "bench_features": "runtime-benchmarks"}, - {"name": "rococo", "package": "rococo-runtime", "path": "polkadot/runtime/rococo", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", "bench_features": "runtime-benchmarks"}, - {"name": "asset-hub-westend", "package": "asset-hub-westend-runtime", "path": "cumulus/parachains/runtimes/assets/asset-hub-westend", "header": "cumulus/file_header.txt", "template": "cumulus/templates/xcm-bench-template.hbs", "bench_features": "runtime-benchmarks"}, + { + "name": "dev", + "package": "kitchensink-runtime", + "path": "substrate/frame", + "header": "substrate/HEADER-APACHE2", + "template": "substrate/.maintain/frame-weight-template.hbs", + "bench_features": "runtime-benchmarks,riscv", + "bench_flags": "--flag1 --flag2" + }, + { + "name": "westend", + "package": "westend-runtime", + "path": "polkadot/runtime/westend", + "header": "polkadot/file_header.txt", + "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--flag3 --flag4" + }, + { + "name": "rococo", + "package": "rococo-runtime", + "path": "polkadot/runtime/rococo", + "header": "polkadot/file_header.txt", + "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "" + }, + { + "name": "asset-hub-westend", + "package": "asset-hub-westend-runtime", + "path": "cumulus/parachains/runtimes/assets/asset-hub-westend", + "header": "cumulus/file_header.txt", + "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--flag7 --flag8" + } ] -def get_mock_bench_output(runtime, pallets, output_path, header, template = None): +def get_mock_bench_output(runtime, pallets, output_path, header, bench_flags, template = None): return f"frame-omni-bencher v1 benchmark pallet --extrinsic=* " \ f"--runtime=target/release/wbuild/{runtime}-runtime/{runtime.replace('-', '_')}_runtime.wasm " \ f"--pallet={pallets} --header={header} " \ @@ -21,7 +53,8 @@ def get_mock_bench_output(runtime, pallets, output_path, header, template = None f"--wasm-execution=compiled " \ f"--steps=50 --repeat=20 --heap-pages=4096 " \ f"{f'--template={template} ' if template else ''}" \ - f"--no-storage-info --no-min-squares --no-median-slopes" + f"--no-storage-info --no-min-squares --no-median-slopes " \ + f"{bench_flags}" class TestCmd(unittest.TestCase): @@ -89,10 +122,29 @@ def test_bench_command_normal_execution_all_runtimes(self): call("forklift cargo build -p rococo-runtime --profile release --features=runtime-benchmarks"), call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"), - call(get_mock_bench_output('kitchensink', 'pallet_balances', './substrate/frame/balances/src/weights.rs', os.path.abspath('substrate/HEADER-APACHE2'), "substrate/.maintain/frame-weight-template.hbs")), - call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', os.path.abspath('polkadot/file_header.txt'))), + call(get_mock_bench_output( + runtime='kitchensink', + pallets='pallet_balances', + output_path='./substrate/frame/balances/src/weights.rs', + header=os.path.abspath('substrate/HEADER-APACHE2'), + bench_flags='--flag1 --flag2', + template="substrate/.maintain/frame-weight-template.hbs" + )), + call(get_mock_bench_output( + runtime='westend', + pallets='pallet_balances', + output_path='./polkadot/runtime/westend/src/weights', + header=os.path.abspath('polkadot/file_header.txt'), + bench_flags='--flag3 --flag4' + )), # skips rococo benchmark - call(get_mock_bench_output('asset-hub-westend', 'pallet_balances', './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', os.path.abspath('cumulus/file_header.txt'))), + call(get_mock_bench_output( + runtime='asset-hub-westend', + pallets='pallet_balances', + output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', + header=os.path.abspath('cumulus/file_header.txt'), + bench_flags='--flag7 --flag8' + )), ] self.mock_system.assert_has_calls(expected_calls, any_order=True) @@ -121,8 +173,20 @@ def test_bench_command_normal_execution(self): call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"), # Westend runtime calls - call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', header_path)), - call(get_mock_bench_output('westend', 'pallet_staking', './polkadot/runtime/westend/src/weights', header_path)), + call(get_mock_bench_output( + runtime='westend', + pallets='pallet_balances', + output_path='./polkadot/runtime/westend/src/weights', + header=header_path, + bench_flags='--flag3 --flag4' + )), + call(get_mock_bench_output( + runtime='westend', + pallets='pallet_staking', + output_path='./polkadot/runtime/westend/src/weights', + header=header_path, + bench_flags='--flag3 --flag4' + )), ] self.mock_system.assert_has_calls(expected_calls, any_order=True) @@ -153,11 +217,12 @@ def test_bench_command_normal_execution_xcm(self): # Westend runtime calls call(get_mock_bench_output( - 'westend', - 'pallet_xcm_benchmarks::generic', - './polkadot/runtime/westend/src/weights/xcm', - header_path, - "polkadot/xcm/pallet-xcm-benchmarks/template.hbs" + runtime='westend', + pallets='pallet_xcm_benchmarks::generic', + output_path='./polkadot/runtime/westend/src/weights/xcm', + header=header_path, + bench_flags='--flag3 --flag4', + template="polkadot/xcm/pallet-xcm-benchmarks/template.hbs" )), ] self.mock_system.assert_has_calls(expected_calls, any_order=True) @@ -188,11 +253,35 @@ def test_bench_command_two_runtimes_two_pallets(self): call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"), call("forklift cargo build -p rococo-runtime --profile release --features=runtime-benchmarks"), # Westend runtime calls - call(get_mock_bench_output('westend', 'pallet_staking', './polkadot/runtime/westend/src/weights', header_path)), - call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', header_path)), + call(get_mock_bench_output( + runtime='westend', + pallets='pallet_staking', + output_path='./polkadot/runtime/westend/src/weights', + header=header_path, + bench_flags='--flag3 --flag4' + )), + call(get_mock_bench_output( + runtime='westend', + pallets='pallet_balances', + output_path='./polkadot/runtime/westend/src/weights', + header=header_path, + bench_flags='--flag3 --flag4' + )), # Rococo runtime calls - call(get_mock_bench_output('rococo', 'pallet_staking', './polkadot/runtime/rococo/src/weights', header_path)), - call(get_mock_bench_output('rococo', 'pallet_balances', './polkadot/runtime/rococo/src/weights', header_path)), + call(get_mock_bench_output( + runtime='rococo', + pallets='pallet_staking', + output_path='./polkadot/runtime/rococo/src/weights', + header=header_path, + bench_flags='' + )), + call(get_mock_bench_output( + runtime='rococo', + pallets='pallet_balances', + output_path='./polkadot/runtime/rococo/src/weights', + header=header_path, + bench_flags='' + )), ] self.mock_system.assert_has_calls(expected_calls, any_order=True) @@ -223,11 +312,12 @@ def test_bench_command_one_dev_runtime(self): call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks,riscv"), # Westend runtime calls call(get_mock_bench_output( - 'kitchensink', - 'pallet_balances', - manifest_dir + "/src/weights.rs", - header_path, - "substrate/.maintain/frame-weight-template.hbs" + runtime='kitchensink', + pallets='pallet_balances', + output_path=manifest_dir + "/src/weights.rs", + header=header_path, + bench_flags='--flag1 --flag2', + template="substrate/.maintain/frame-weight-template.hbs" )), ] self.mock_system.assert_has_calls(expected_calls, any_order=True) @@ -257,10 +347,11 @@ def test_bench_command_one_cumulus_runtime(self): call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"), # Asset-hub-westend runtime calls call(get_mock_bench_output( - 'asset-hub-westend', - 'pallet_assets', - './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', - header_path + runtime='asset-hub-westend', + pallets='pallet_assets', + output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', + header=header_path, + bench_flags='--flag7 --flag8' )), ] @@ -291,17 +382,19 @@ def test_bench_command_one_cumulus_runtime_xcm(self): call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"), # Asset-hub-westend runtime calls call(get_mock_bench_output( - 'asset-hub-westend', - 'pallet_xcm_benchmarks::generic', - './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights/xcm', - header_path, - "cumulus/templates/xcm-bench-template.hbs" + runtime='asset-hub-westend', + pallets='pallet_xcm_benchmarks::generic', + output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights/xcm', + header=header_path, + bench_flags='--flag7 --flag8', + template="cumulus/templates/xcm-bench-template.hbs" )), call(get_mock_bench_output( - 'asset-hub-westend', - 'pallet_assets', - './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', - header_path + runtime='asset-hub-westend', + pallets='pallet_assets', + output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', + header=header_path, + bench_flags='--flag7 --flag8' )), ] diff --git a/.github/workflows/check-frame-omni-bencher.yml b/.github/workflows/check-frame-omni-bencher.yml index 9b01311aa69c..935fc080c4ea 100644 --- a/.github/workflows/check-frame-omni-bencher.yml +++ b/.github/workflows/check-frame-omni-bencher.yml @@ -16,14 +16,11 @@ env: ARTIFACTS_NAME: frame-omni-bencher-artifacts jobs: - preflight: - # TODO: remove once migration is complete or this workflow is fully stable - if: contains(github.event.label.name, 'GHA-migration') uses: ./.github/workflows/reusable-preflight.yml quick-benchmarks-omni: - runs-on: ${{ needs.preflight.outputs.RUNNER }} + runs-on: ${{ needs.preflight.outputs.RUNNER_BENCHMARK }} needs: [preflight] if: ${{ needs.preflight.outputs.changes_rust }} env: @@ -31,6 +28,7 @@ jobs: RUST_BACKTRACE: "full" WASM_BUILD_NO_COLOR: 1 WASM_BUILD_RUSTFLAGS: "-C debug-assertions" + RUST_LOG: "frame_omni_bencher=info,polkadot_sdk_frame=info" timeout-minutes: 30 container: image: ${{ needs.preflight.outputs.IMAGE }} @@ -42,33 +40,41 @@ jobs: forklift cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks forklift cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet + runtime-matrix: + runs-on: ubuntu-latest + needs: [preflight] + if: ${{ needs.preflight.outputs.changes_rust }} + timeout-minutes: 30 + outputs: + runtime: ${{ steps.runtime.outputs.runtime }} + container: + image: ${{ needs.preflight.outputs.IMAGE }} + name: Extract runtimes from matrix + steps: + - uses: actions/checkout@v4 + - id: runtime + run: | + RUNTIMES=$(jq '[.[] | select(.package != null)]' .github/workflows/runtimes-matrix.json) + + RUNTIMES=$(echo $RUNTIMES | jq -c .) + echo "runtime=$RUNTIMES" + echo "runtime=$RUNTIMES" >> $GITHUB_OUTPUT + run-frame-omni-bencher: - runs-on: ${{ needs.preflight.outputs.RUNNER }} - needs: [preflight] # , build-frame-omni-bencher ] + runs-on: ${{ needs.preflight.outputs.RUNNER_BENCHMARK }} + needs: [preflight, runtime-matrix] if: ${{ needs.preflight.outputs.changes_rust }} timeout-minutes: 30 strategy: fail-fast: false # keep running other workflows even if one fails, to see the logs of all possible failures matrix: - runtime: - [ - westend-runtime, - rococo-runtime, - asset-hub-rococo-runtime, - asset-hub-westend-runtime, - bridge-hub-rococo-runtime, - bridge-hub-westend-runtime, - collectives-westend-runtime, - coretime-rococo-runtime, - coretime-westend-runtime, - people-rococo-runtime, - people-westend-runtime, - glutton-westend-runtime, - ] + runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }} container: image: ${{ needs.preflight.outputs.IMAGE }} env: - PACKAGE_NAME: ${{ matrix.runtime }} + PACKAGE_NAME: ${{ matrix.runtime.package }} + FLAGS: ${{ matrix.runtime.bench_flags }} + RUST_LOG: "frame_omni_bencher=info,polkadot_sdk_frame=info" steps: - name: Checkout uses: actions/checkout@v4 @@ -77,10 +83,13 @@ jobs: run: | RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm RUNTIME_BLOB_PATH=./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME - forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features runtime-benchmarks + forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features=${{ matrix.runtime.bench_features }} --quiet echo "Running short benchmarking for PACKAGE_NAME=$PACKAGE_NAME and RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH" ls -lrt $RUNTIME_BLOB_PATH - ./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1 + + cmd="./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1 $FLAGS" + echo "Running command: $cmd" + eval "$cmd" confirm-frame-omni-benchers-passed: runs-on: ubuntu-latest name: All benchmarks passed diff --git a/.github/workflows/runtimes-matrix.json b/.github/workflows/runtimes-matrix.json index 50fa44b1b073..e4e3a2dbe6d1 100644 --- a/.github/workflows/runtimes-matrix.json +++ b/.github/workflows/runtimes-matrix.json @@ -6,6 +6,7 @@ "header": "substrate/HEADER-APACHE2", "template": "substrate/.maintain/frame-weight-template.hbs", "bench_features": "runtime-benchmarks,riscv", + "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic,pallet_nomination_pools,pallet_remark,pallet_transaction_storage", "uri": null, "is_relay": false }, @@ -15,8 +16,9 @@ "path": "polkadot/runtime/westend", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", - "uri": "wss://try-runtime-westend.polkadot.io:443", + "bench_flags": "", "bench_features": "runtime-benchmarks", + "uri": "wss://try-runtime-westend.polkadot.io:443", "is_relay": true }, { @@ -27,6 +29,7 @@ "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", "uri": "wss://try-runtime-rococo.polkadot.io:443", "bench_features": "runtime-benchmarks", + "bench_flags": "", "is_relay": true }, { @@ -36,6 +39,7 @@ "header": "cumulus/file_header.txt", "template": "cumulus/templates/xcm-bench-template.hbs", "bench_features": "runtime-benchmarks", + "bench_flags": "", "uri": "wss://westend-asset-hub-rpc.polkadot.io:443", "is_relay": false }, @@ -44,8 +48,9 @@ "package": "asset-hub-rococo-runtime", "path": "cumulus/parachains/runtimes/assets/asset-hub-rococo", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "", "uri": "wss://rococo-asset-hub-rpc.polkadot.io:443", "is_relay": false }, @@ -54,8 +59,9 @@ "package": "bridge-hub-rococo-runtime", "path": "cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "", "uri": "wss://rococo-bridge-hub-rpc.polkadot.io:443", "is_relay": false }, @@ -64,8 +70,9 @@ "package": "bridge-hub-rococo-runtime", "path": "cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "", "uri": "wss://westend-bridge-hub-rpc.polkadot.io:443", "is_relay": false }, @@ -74,8 +81,9 @@ "package": "collectives-westend-runtime", "path": "cumulus/parachains/runtimes/collectives/collectives-westend", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "", "uri": "wss://westend-collectives-rpc.polkadot.io:443" }, { @@ -83,8 +91,9 @@ "package": "contracts-rococo-runtime", "path": "cumulus/parachains/runtimes/contracts/contracts-rococo", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm", "uri": "wss://rococo-contracts-rpc.polkadot.io:443", "is_relay": false }, @@ -93,8 +102,9 @@ "package": "coretime-rococo-runtime", "path": "cumulus/parachains/runtimes/coretime/coretime-rococo", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic", "uri": "wss://rococo-coretime-rpc.polkadot.io:443", "is_relay": false }, @@ -103,8 +113,9 @@ "package": "coretime-westend-runtime", "path": "cumulus/parachains/runtimes/coretime/coretime-westend", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic", "uri": "wss://westend-coretime-rpc.polkadot.io:443", "is_relay": false }, @@ -113,8 +124,9 @@ "package": "glutton-westend-runtime", "path": "cumulus/parachains/runtimes/gluttons/glutton-westend", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--genesis-builder-policy=none", "uri": null, "is_relay": false }, @@ -123,8 +135,9 @@ "package": "people-rococo-runtime", "path": "cumulus/parachains/runtimes/people/people-rococo", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic", "uri": "wss://rococo-people-rpc.polkadot.io:443", "is_relay": false }, @@ -133,8 +146,9 @@ "package": "people-westend-runtime", "path": "cumulus/parachains/runtimes/people/people-westend", "header": "cumulus/file_header.txt", - "bench_features": "runtime-benchmarks", "template": "cumulus/templates/xcm-bench-template.hbs", + "bench_features": "runtime-benchmarks", + "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic", "uri": "wss://westend-people-rpc.polkadot.io:443", "is_relay": false } diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml index b02d857f3f24..2a7e2e7559a0 100644 --- a/.gitlab/pipeline/zombienet.yml +++ b/.gitlab/pipeline/zombienet.yml @@ -1,12 +1,14 @@ .zombienet-refs: extends: .build-refs variables: - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.113" + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.114" PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics" DEBUG: "zombie,zombie::network-node,zombie::kube::client::logs" ZOMBIE_PROVIDER: "k8s" RUST_LOG: "info,zombienet_orchestrator=debug" RUN_IN_CI: "1" + KUBERNETES_CPU_REQUEST: "512m" + KUBERNETES_MEMORY_REQUEST: "512M" timeout: 60m include: diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 0da95f224cfc..33191d99941d 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -94,7 +94,7 @@ zombienet-polkadot-functional-0002-parachains-disputes: --local-dir="${LOCAL_DIR}/functional" --test="0002-parachains-disputes.zndsl" -zombienet-polkadot-functional-0003-beefy-and-mmr: +.zombienet-polkadot-functional-0003-beefy-and-mmr: extends: - .zombienet-polkadot-common script: @@ -190,7 +190,7 @@ zombienet-polkadot-elastic-scaling-0002-elastic-scaling-doesnt-break-parachains: --local-dir="${LOCAL_DIR}/elastic_scaling" --test="0002-elastic-scaling-doesnt-break-parachains.zndsl" -zombienet-polkadot-functional-0012-spam-statement-distribution-requests: +.zombienet-polkadot-functional-0012-spam-statement-distribution-requests: extends: - .zombienet-polkadot-common script: @@ -277,7 +277,7 @@ zombienet-polkadot-smoke-0002-parachains-parachains-upgrade-smoke: --local-dir="${LOCAL_DIR}/smoke" --test="0002-parachains-upgrade-smoke-test.zndsl" -zombienet-polkadot-smoke-0003-deregister-register-validator: +.zombienet-polkadot-smoke-0003-deregister-register-validator: extends: - .zombienet-polkadot-common script: diff --git a/.gitlab/pipeline/zombienet/substrate.yml b/.gitlab/pipeline/zombienet/substrate.yml index d4ffa34f3395..030a0a3f50a9 100644 --- a/.gitlab/pipeline/zombienet/substrate.yml +++ b/.gitlab/pipeline/zombienet/substrate.yml @@ -70,7 +70,7 @@ zombienet-substrate-0001-basic-warp-sync: --local-dir="${LOCAL_DIR}/0001-basic-warp-sync" --test="test-warp-sync.zndsl" -zombienet-substrate-0002-validators-warp-sync: +.zombienet-substrate-0002-validators-warp-sync: extends: - .zombienet-substrate-warp-sync-common before_script: diff --git a/Cargo.lock b/Cargo.lock index 4d5565e433b7..34e30d79e98b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -900,7 +900,7 @@ dependencies = [ "parity-scale-codec", "polkadot-parachain-primitives", "polkadot-runtime-common", - "primitive-types", + "primitive-types 0.13.1", "rococo-runtime-constants", "scale-info", "serde_json", @@ -1033,7 +1033,7 @@ dependencies = [ "parity-scale-codec", "polkadot-parachain-primitives", "polkadot-runtime-common", - "primitive-types", + "primitive-types 0.13.1", "scale-info", "serde_json", "snowbridge-router-primitives", @@ -2048,7 +2048,6 @@ dependencies = [ "frame-system", "hex", "parity-scale-codec", - "parity-util-mem", "scale-info", "serde", "sp-core 28.0.0", @@ -5756,9 +5755,9 @@ dependencies = [ [[package]] name = "ethabi-decode" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d398648d65820a727d6a81e58b962f874473396a047e4c30bafe3240953417" +checksum = "f9af52ec57c5147716872863c2567c886e7d62f539465b94352dbc0108fe5293" dependencies = [ "ethereum-types", "tiny-keccak", @@ -5766,33 +5765,33 @@ dependencies = [ [[package]] name = "ethbloom" -version = "0.13.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60" +checksum = "8c321610643004cf908ec0f5f2aa0d8f1f8e14b540562a2887a1111ff1ecbf7b" dependencies = [ "crunchy", "fixed-hash", - "impl-codec", + "impl-codec 0.7.0", "impl-rlp", - "impl-serde", + "impl-serde 0.5.0", "scale-info", "tiny-keccak", ] [[package]] name = "ethereum-types" -version = "0.14.1" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d215cbf040552efcbe99a38372fe80ab9d00268e20012b79fcd0f073edd8ee" +checksum = "1ab15ed80916029f878e0267c3a9f92b67df55e79af370bf66199059ae2b4ee3" dependencies = [ "ethbloom", "fixed-hash", - "impl-codec", + "impl-codec 0.7.0", "impl-rlp", - "impl-serde", - "primitive-types", + "impl-serde 0.5.0", + "primitive-types 0.13.1", "scale-info", - "uint", + "uint 0.10.0", ] [[package]] @@ -7610,6 +7609,15 @@ dependencies = [ "parity-scale-codec", ] +[[package]] +name = "impl-codec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67aa010c1e3da95bf151bd8b4c059b2ed7e75387cdb969b4f8f2723a43f9941" +dependencies = [ + "parity-scale-codec", +] + [[package]] name = "impl-num-traits" version = "0.1.2" @@ -7618,16 +7626,27 @@ checksum = "951641f13f873bff03d4bf19ae8bec531935ac0ac2cc775f84d7edfdcfed3f17" dependencies = [ "integer-sqrt", "num-traits", - "uint", + "uint 0.9.5", +] + +[[package]] +name = "impl-num-traits" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d15461ab0dcc56706adf266158acbc44ccf719bf7d0af30705f58b90a4b8c" +dependencies = [ + "integer-sqrt", + "num-traits", + "uint 0.10.0", ] [[package]] name = "impl-rlp" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +checksum = "54ed8ad1f3877f7e775b8cbf30ed1bd3209a95401817f19a0eb4402d13f8cf90" dependencies = [ - "rlp", + "rlp 0.6.1", ] [[package]] @@ -7639,6 +7658,15 @@ dependencies = [ "serde", ] +[[package]] +name = "impl-serde" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a143eada6a1ec4aefa5049037a26a6d597bfd64f8c026d07b77133e02b7dd0b" +dependencies = [ + "serde", +] + [[package]] name = "impl-trait-for-tuples" version = "0.2.2" @@ -8366,7 +8394,7 @@ dependencies = [ "pallet-example-tasks", "parity-scale-codec", "polkadot-sdk", - "primitive-types", + "primitive-types 0.13.1", "scale-info", "serde_json", "static_assertions", @@ -8779,7 +8807,7 @@ dependencies = [ "sha2 0.10.8", "smallvec", "thiserror", - "uint", + "uint 0.9.5", "unsigned-varint 0.7.2", "void", ] @@ -9242,7 +9270,7 @@ dependencies = [ "tokio-tungstenite", "tokio-util", "tracing", - "uint", + "uint 0.9.5", "unsigned-varint 0.8.0", "url", "x25519-dalek", @@ -9277,15 +9305,6 @@ dependencies = [ "value-bag", ] -[[package]] -name = "lru" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909" -dependencies = [ - "hashbrown 0.12.3", -] - [[package]] name = "lru" version = "0.11.0" @@ -10628,7 +10647,7 @@ dependencies = [ "pallet-assets", "pallet-balances", "parity-scale-codec", - "primitive-types", + "primitive-types 0.13.1", "scale-info", "sp-api 26.0.0", "sp-arithmetic 23.0.0", @@ -10649,7 +10668,7 @@ dependencies = [ "pallet-assets", "pallet-balances", "parity-scale-codec", - "primitive-types", + "primitive-types 0.13.1", "scale-info", "sp-arithmetic 23.0.0", "sp-core 28.0.0", @@ -12313,7 +12332,7 @@ dependencies = [ "polkavm 0.12.0", "polkavm-common 0.12.0", "pretty_assertions", - "rlp", + "rlp 0.6.1", "scale-info", "serde", "sp-api 26.0.0", @@ -13223,7 +13242,7 @@ checksum = "4e69bf016dc406eff7d53a7d3f7cf1c2e72c82b9088aac1118591e36dd2cd3e9" dependencies = [ "bitcoin_hashes 0.13.0", "rand", - "rand_core 0.6.4", + "rand_core 0.5.1", "serde", "unicode-normalization", ] @@ -13281,35 +13300,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "parity-util-mem" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d32c34f4f5ca7f9196001c0aba5a1f9a5a12382c8944b8b0f90233282d1e8f8" -dependencies = [ - "cfg-if", - "ethereum-types", - "hashbrown 0.12.3", - "impl-trait-for-tuples", - "lru 0.8.1", - "parity-util-mem-derive", - "parking_lot 0.12.3", - "primitive-types", - "smallvec", - "winapi", -] - -[[package]] -name = "parity-util-mem-derive" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" -dependencies = [ - "proc-macro2 1.0.86", - "syn 1.0.109", - "synstructure 0.12.6", -] - [[package]] name = "parity-wasm" version = "0.45.0" @@ -13481,7 +13471,7 @@ dependencies = [ "polkadot-parachain-primitives", "polkadot-primitives", "polkadot-runtime-common", - "primitive-types", + "primitive-types 0.12.2", "scale-info", "smallvec", "sp-api 26.0.0", @@ -16523,12 +16513,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ "fixed-hash", - "impl-codec", - "impl-num-traits", + "impl-codec 0.6.0", + "impl-num-traits 0.1.2", + "impl-serde 0.4.0", + "scale-info", + "uint 0.9.5", +] + +[[package]] +name = "primitive-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15600a7d856470b7d278b3fe0e311fe28c2526348549f8ef2ff7db3299c87f5" +dependencies = [ + "fixed-hash", + "impl-codec 0.7.0", + "impl-num-traits 0.2.0", "impl-rlp", - "impl-serde", + "impl-serde 0.5.0", "scale-info", - "uint", + "uint 0.10.0", ] [[package]] @@ -17640,6 +17644,16 @@ dependencies = [ "rustc-hex", ] +[[package]] +name = "rlp" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa24e92bb2a83198bb76d661a71df9f7076b8c420b8696e4d3d97d50d94479e3" +dependencies = [ + "bytes", + "rustc-hex", +] + [[package]] name = "rocksdb" version = "0.21.0" @@ -17950,10 +17964,10 @@ dependencies = [ "num-bigint", "num-traits", "parity-scale-codec", - "primitive-types", + "primitive-types 0.12.2", "proptest", "rand", - "rlp", + "rlp 0.5.2", "ruint-macro", "serde", "valuable", @@ -19883,7 +19897,7 @@ checksum = "e98f3262c250d90e700bb802eb704e1f841e03331c2eb815e46516c4edbf5b27" dependencies = [ "derive_more", "parity-scale-codec", - "primitive-types", + "primitive-types 0.12.2", "scale-bits", "scale-decode-derive", "scale-type-resolver", @@ -19910,7 +19924,7 @@ checksum = "4ba0b9c48dc0eb20c60b083c29447c0c4617cb7c4a4c9fef72aa5c5bc539e15e" dependencies = [ "derive_more", "parity-scale-codec", - "primitive-types", + "primitive-types 0.12.2", "scale-bits", "scale-encode-derive", "scale-type-resolver", @@ -20872,7 +20886,7 @@ dependencies = [ "hex", "hex-literal", "parity-scale-codec", - "rlp", + "rlp 0.6.1", "scale-info", "serde", "snowbridge-ethereum", @@ -20920,7 +20934,7 @@ dependencies = [ "parity-bytes", "parity-scale-codec", "rand", - "rlp", + "rlp 0.6.1", "scale-info", "serde", "serde-big-array", @@ -21478,7 +21492,7 @@ dependencies = [ "integer-sqrt", "num-traits", "parity-scale-codec", - "primitive-types", + "primitive-types 0.13.1", "rand", "scale-info", "serde", @@ -21718,7 +21732,7 @@ dependencies = [ "futures", "hash-db", "hash256-std-hasher", - "impl-serde", + "impl-serde 0.4.0", "itertools 0.11.0", "k256", "libsecp256k1", @@ -21728,7 +21742,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.12.3", "paste", - "primitive-types", + "primitive-types 0.13.1", "rand", "regex", "scale-info", @@ -21767,7 +21781,7 @@ dependencies = [ "futures", "hash-db", "hash256-std-hasher", - "impl-serde", + "impl-serde 0.4.0", "itertools 0.10.5", "k256", "libsecp256k1", @@ -21777,7 +21791,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.12.3", "paste", - "primitive-types", + "primitive-types 0.12.2", "rand", "scale-info", "schnorrkel 0.11.4", @@ -21814,7 +21828,7 @@ dependencies = [ "futures", "hash-db", "hash256-std-hasher", - "impl-serde", + "impl-serde 0.4.0", "itertools 0.10.5", "k256", "libsecp256k1", @@ -21824,7 +21838,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.12.3", "paste", - "primitive-types", + "primitive-types 0.12.2", "rand", "scale-info", "schnorrkel 0.11.4", @@ -21861,7 +21875,7 @@ dependencies = [ "futures", "hash-db", "hash256-std-hasher", - "impl-serde", + "impl-serde 0.4.0", "itertools 0.11.0", "k256", "libsecp256k1", @@ -21871,7 +21885,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.12.3", "paste", - "primitive-types", + "primitive-types 0.12.2", "rand", "scale-info", "schnorrkel 0.11.4", @@ -22516,7 +22530,7 @@ dependencies = [ "bytes", "impl-trait-for-tuples", "parity-scale-codec", - "primitive-types", + "primitive-types 0.12.2", "sp-externalities 0.19.0", "sp-runtime-interface-proc-macro 11.0.0", "sp-std 8.0.0", @@ -22534,7 +22548,7 @@ dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", "polkavm-derive 0.9.1", - "primitive-types", + "primitive-types 0.13.1", "rustversion", "sp-core 28.0.0", "sp-externalities 0.25.0", @@ -22560,7 +22574,7 @@ dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", "polkavm-derive 0.8.0", - "primitive-types", + "primitive-types 0.12.2", "sp-externalities 0.27.0", "sp-runtime-interface-proc-macro 18.0.0", "sp-std 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -22580,7 +22594,7 @@ dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", "polkavm-derive 0.9.1", - "primitive-types", + "primitive-types 0.12.2", "sp-externalities 0.28.0", "sp-runtime-interface-proc-macro 18.0.0", "sp-std 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -22821,7 +22835,7 @@ name = "sp-storage" version = "13.0.0" source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "ref-cast", "serde", @@ -22833,7 +22847,7 @@ dependencies = [ name = "sp-storage" version = "19.0.0" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "ref-cast", "serde", @@ -22846,7 +22860,7 @@ version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8dba5791cb3978e95daf99dad919ecb3ec35565604e88cd38d805d9d4981e8bd" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "ref-cast", "serde", @@ -22860,7 +22874,7 @@ version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99c82989b3a4979a7e1ad848aad9f5d0b4388f1f454cc131766526601ab9e8f8" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "ref-cast", "serde", @@ -23061,7 +23075,7 @@ dependencies = [ name = "sp-version" version = "29.0.0" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "parity-wasm", "scale-info", @@ -23079,7 +23093,7 @@ version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff74bf12b4f7d29387eb1caeec5553209a505f90a2511d2831143b970f89659" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "parity-wasm", "scale-info", @@ -23413,7 +23427,7 @@ dependencies = [ "polkadot-primitives", "polkadot-runtime-parachains", "polkadot-test-runtime", - "primitive-types", + "primitive-types 0.13.1", "scale-info", "sp-arithmetic 23.0.0", "sp-core 28.0.0", @@ -24038,11 +24052,11 @@ dependencies = [ "frame-metadata 16.0.0", "futures", "hex", - "impl-serde", + "impl-serde 0.4.0", "instant", "jsonrpsee 0.22.5", "parity-scale-codec", - "primitive-types", + "primitive-types 0.12.2", "reconnecting-jsonrpsee-ws-client", "scale-bits", "scale-decode", @@ -24095,9 +24109,9 @@ dependencies = [ "frame-metadata 16.0.0", "hashbrown 0.14.5", "hex", - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", - "primitive-types", + "primitive-types 0.12.2", "scale-bits", "scale-decode", "scale-encode", @@ -25448,6 +25462,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "uint" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909988d098b2f738727b161a106cfc7cab00c539c2687a8836f8e565976fb53e" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + [[package]] name = "unarray" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index bd37dee51460..a52cdb99b0e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -741,8 +741,8 @@ env_logger = { version = "0.11.2" } environmental = { version = "1.1.4", default-features = false } equivocation-detector = { path = "bridges/relays/equivocation" } ethabi = { version = "1.0.0", default-features = false, package = "ethabi-decode" } -ethbloom = { version = "0.13.0", default-features = false } -ethereum-types = { version = "0.14.1", default-features = false } +ethbloom = { version = "0.14.1", default-features = false } +ethereum-types = { version = "0.15.1", default-features = false } exit-future = { version = "0.2.0" } expander = { version = "2.0.0" } fatality = { version = "0.1.1" } @@ -1003,7 +1003,6 @@ parachains-relay = { path = "bridges/relays/parachains" } parachains-runtimes-test-utils = { path = "cumulus/parachains/runtimes/test-utils", default-features = false } parity-bytes = { version = "0.1.2", default-features = false } parity-db = { version = "0.4.12" } -parity-util-mem = { version = "0.12.0" } parity-wasm = { version = "0.45.0" } parking_lot = { version = "0.12.1", default-features = false } partial_sort = { version = "0.2.0" } @@ -1078,7 +1077,7 @@ polkavm-derive = "0.9.1" polkavm-linker = "0.9.2" portpicker = { version = "0.1.1" } pretty_assertions = { version = "1.3.0" } -primitive-types = { version = "0.12.1", default-features = false } +primitive-types = { version = "0.13.1", default-features = false, features = ["num-traits"] } proc-macro-crate = { version = "3.0.0" } proc-macro-warning = { version = "1.0.0", default-features = false } proc-macro2 = { version = "1.0.86" } @@ -1106,7 +1105,7 @@ relay-substrate-client = { path = "bridges/relays/client-substrate" } relay-utils = { path = "bridges/relays/utils" } remote-externalities = { path = "substrate/utils/frame/remote-externalities", default-features = false, package = "frame-remote-externalities" } reqwest = { version = "0.11", default-features = false } -rlp = { version = "0.5.2", default-features = false } +rlp = { version = "0.6.1", default-features = false } rococo-emulated-chain = { path = "cumulus/parachains/integration-tests/emulated/chains/relays/rococo" } rococo-parachain-runtime = { path = "cumulus/parachains/runtimes/testing/rococo-parachain" } rococo-runtime = { path = "polkadot/runtime/rococo" } diff --git a/bridges/primitives/polkadot-core/Cargo.toml b/bridges/primitives/polkadot-core/Cargo.toml index 366ee7aa948e..295fb281e9bb 100644 --- a/bridges/primitives/polkadot-core/Cargo.toml +++ b/bridges/primitives/polkadot-core/Cargo.toml @@ -12,7 +12,6 @@ workspace = true [dependencies] codec = { features = ["derive"], workspace = true } -parity-util-mem = { optional = true, workspace = true } scale-info = { features = ["derive"], workspace = true } serde = { optional = true, features = [ "derive", @@ -42,7 +41,6 @@ std = [ "codec/std", "frame-support/std", "frame-system/std", - "parity-util-mem", "scale-info/std", "serde", "sp-core/std", diff --git a/bridges/primitives/polkadot-core/src/parachains.rs b/bridges/primitives/polkadot-core/src/parachains.rs index d54ee108386e..a8b1cf6eebf4 100644 --- a/bridges/primitives/polkadot-core/src/parachains.rs +++ b/bridges/primitives/polkadot-core/src/parachains.rs @@ -32,9 +32,6 @@ use sp_std::vec::Vec; #[cfg(feature = "std")] use serde::{Deserialize, Serialize}; -#[cfg(feature = "std")] -use parity_util_mem::MallocSizeOf; - /// Parachain id. /// /// This is an equivalent of the `polkadot_parachain_primitives::Id`, which is a compact-encoded @@ -71,7 +68,7 @@ impl From for ParaId { #[derive( PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug, TypeInfo, Default, )] -#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash))] pub struct ParaHead(pub Vec); impl ParaHead { diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs index e8cc9d02fb0e..40801f66a47b 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs @@ -72,7 +72,10 @@ impl Config for Runtime { type MaxDebugBufferLen = ConstU32<{ 2 * 1024 * 1024 }>; type MaxDelegateDependencies = ConstU32<32>; type CodeHashLockupDepositPercent = CodeHashLockupDepositPercent; - type Migrations = (pallet_contracts::migration::v16::Migration,); + #[cfg(not(feature = "runtime-benchmarks"))] + type Migrations = (); + #[cfg(feature = "runtime-benchmarks")] + type Migrations = pallet_contracts::migration::codegen::BenchMigrations; type RuntimeHoldReason = RuntimeHoldReason; type Debug = (); type Environment = (); diff --git a/prdoc/pr_5872.prdoc b/prdoc/pr_5872.prdoc new file mode 100644 index 000000000000..cf4f0b24f8db --- /dev/null +++ b/prdoc/pr_5872.prdoc @@ -0,0 +1,13 @@ +title: '[omni-bencher] Make all runtimes work' +doc: +- audience: Runtime Dev + description: |- + Changes: + - Add `--exclude-pallets` to exclude some pallets from runtimes where we dont have genesis presets yet + - Make `--genesis-builder-policy=none` work with `--runtime` + - CI: Run the frame-omni-bencher for all runtimes +crates: +- name: frame-benchmarking-cli + bump: major +- name: contracts-rococo-runtime + bump: patch diff --git a/prdoc/pr_5886.prdoc b/prdoc/pr_5886.prdoc new file mode 100644 index 000000000000..f5e597281197 --- /dev/null +++ b/prdoc/pr_5886.prdoc @@ -0,0 +1,18 @@ +title: Bump some dependencies +doc: +- audience: Runtime Dev + description: |- + This bumps `ethbloom`, `ethereum-types`, `primitive-types` and `rlp` to their latest version. + + Fixes: https://github.com/paritytech/polkadot-sdk/issues/5870 +crates: +- name: sc-consensus-babe + bump: patch +- name: pallet-babe + bump: patch +- name: pallet-revive + bump: patch +- name: sp-runtime + bump: patch +- name: bp-polkadot-core + bump: major diff --git a/substrate/client/consensus/babe/src/authorship.rs b/substrate/client/consensus/babe/src/authorship.rs index 57ee706a04f6..06394b84f470 100644 --- a/substrate/client/consensus/babe/src/authorship.rs +++ b/substrate/client/consensus/babe/src/authorship.rs @@ -108,7 +108,8 @@ pub(super) fn secondary_slot_author( return None } - let rand = U256::from((randomness, slot).using_encoded(sp_crypto_hashing::blake2_256)); + let rand = + U256::from_little_endian(&(randomness, slot).using_encoded(sp_crypto_hashing::blake2_256)); let authorities_len = U256::from(authorities.len()); let idx = rand % authorities_len; diff --git a/substrate/frame/babe/src/mock.rs b/substrate/frame/babe/src/mock.rs index 4e4052b2b566..d416e31b25f7 100644 --- a/substrate/frame/babe/src/mock.rs +++ b/substrate/frame/babe/src/mock.rs @@ -291,7 +291,7 @@ pub fn new_test_ext_with_pairs( authorities_len: usize, ) -> (Vec, sp_io::TestExternalities) { let pairs = (0..authorities_len) - .map(|i| AuthorityPair::from_seed(&U256::from(i).into())) + .map(|i| AuthorityPair::from_seed(&U256::from(i).to_little_endian())) .collect::>(); let public = pairs.iter().map(|p| p.public()).collect(); diff --git a/substrate/frame/revive/src/wasm/runtime.rs b/substrate/frame/revive/src/wasm/runtime.rs index 485abbfda225..245c91278a7f 100644 --- a/substrate/frame/revive/src/wasm/runtime.rs +++ b/substrate/frame/revive/src/wasm/runtime.rs @@ -46,9 +46,7 @@ const MAX_DECODE_NESTING: u32 = 256; /// Encode a `U256` into a 32 byte buffer. fn as_bytes(u: U256) -> [u8; 32] { - let mut bytes = [0u8; 32]; - u.to_little_endian(&mut bytes); - bytes + u.to_little_endian() } #[derive(Clone, Copy)] diff --git a/substrate/frame/sassafras/src/mock.rs b/substrate/frame/sassafras/src/mock.rs index f145bffa3a05..d2b329e8a2ba 100644 --- a/substrate/frame/sassafras/src/mock.rs +++ b/substrate/frame/sassafras/src/mock.rs @@ -89,7 +89,7 @@ pub fn new_test_ext_with_pairs( with_ring_context: bool, ) -> (Vec, sp_io::TestExternalities) { let pairs = (0..authorities_len) - .map(|i| AuthorityPair::from_seed(&U256::from(i).into())) + .map(|i| AuthorityPair::from_seed(&U256::from(i).to_big_endian())) .collect::>(); let authorities: Vec<_> = pairs.iter().map(|p| p.public()).collect(); diff --git a/substrate/primitives/runtime/src/testing.rs b/substrate/primitives/runtime/src/testing.rs index b4aeda5a0e7a..a4ce4b5fc1a0 100644 --- a/substrate/primitives/runtime/src/testing.rs +++ b/substrate/primitives/runtime/src/testing.rs @@ -29,10 +29,7 @@ use crate::{ ApplyExtrinsicResultWithInfo, KeyTypeId, }; use serde::{de::Error as DeError, Deserialize, Deserializer, Serialize, Serializer}; -use sp_core::{ - crypto::{key_types, ByteArray, CryptoType, Dummy}, - U256, -}; +use sp_core::crypto::{key_types, ByteArray, CryptoType, Dummy}; pub use sp_core::{sr25519, H256}; use std::{ cell::RefCell, @@ -79,7 +76,8 @@ impl From for u64 { impl UintAuthorityId { /// Convert this authority ID into a public key. pub fn to_public_key(&self) -> T { - let bytes: [u8; 32] = U256::from(self.0).into(); + let mut bytes = [0u8; 32]; + bytes[0..8].copy_from_slice(&self.0.to_le_bytes()); T::from_slice(&bytes).unwrap() } } diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs index f1499f41c2ea..f33348190577 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -36,7 +36,7 @@ use sp_core::{ testing::{TestOffchainExt, TestTransactionPoolExt}, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, }, - traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt}, + traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt, WrappedRuntimeCode}, Hasher, }; use sp_externalities::Extensions; @@ -47,6 +47,7 @@ use sp_storage::{well_known_keys::CODE, Storage}; use sp_trie::{proof_size_extension::ProofSizeExt, recorder::Recorder}; use sp_wasm_interface::HostFunctions; use std::{ + borrow::Cow, collections::{BTreeMap, BTreeSet, HashMap}, fmt::Debug, fs, @@ -368,12 +369,12 @@ impl PalletCmd { "dispatch a benchmark", ) { Err(e) => { - log::error!("Error executing and verifying runtime benchmark: {}", e); + log::error!(target: LOG_TARGET, "Error executing and verifying runtime benchmark: {}", e); failed.push((pallet.clone(), extrinsic.clone())); continue 'outer }, Ok(Err(e)) => { - log::error!("Error executing and verifying runtime benchmark: {}", e); + log::error!(target: LOG_TARGET, "Error executing and verifying runtime benchmark: {}", e); failed.push((pallet.clone(), extrinsic.clone())); continue 'outer }, @@ -408,12 +409,12 @@ impl PalletCmd { "dispatch a benchmark", ) { Err(e) => { - log::error!("Error executing runtime benchmark: {}", e); + log::error!(target: LOG_TARGET, "Error executing runtime benchmark: {}", e); failed.push((pallet.clone(), extrinsic.clone())); continue 'outer }, Ok(Err(e)) => { - log::error!("Benchmark {pallet}::{extrinsic} failed: {e}",); + log::error!(target: LOG_TARGET, "Benchmark {pallet}::{extrinsic} failed: {e}",); failed.push((pallet.clone(), extrinsic.clone())); continue 'outer }, @@ -502,33 +503,28 @@ impl PalletCmd { } fn select_benchmarks_to_run(&self, list: Vec) -> Result> { - let pallet = self.pallet.clone().unwrap_or_default(); - let pallet = pallet.as_bytes(); - let extrinsic = self.extrinsic.clone().unwrap_or_default(); let extrinsic_split: Vec<&str> = extrinsic.split(',').collect(); let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect(); // Use the benchmark list and the user input to determine the set of benchmarks to run. let mut benchmarks_to_run = Vec::new(); - list.iter() - .filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..]) - .for_each(|item| { - for benchmark in &item.benchmarks { - let benchmark_name = &benchmark.name; - if extrinsic.is_empty() || - extrinsic.as_bytes() == &b"*"[..] || - extrinsics.contains(&&benchmark_name[..]) - { - benchmarks_to_run.push(( - item.pallet.clone(), - benchmark.name.clone(), - benchmark.components.clone(), - benchmark.pov_modes.clone(), - )) - } + list.iter().filter(|item| self.pallet_selected(&item.pallet)).for_each(|item| { + for benchmark in &item.benchmarks { + let benchmark_name = &benchmark.name; + if extrinsic.is_empty() || + extrinsic.as_bytes() == &b"*"[..] || + extrinsics.contains(&&benchmark_name[..]) + { + benchmarks_to_run.push(( + item.pallet.clone(), + benchmark.name.clone(), + benchmark.components.clone(), + benchmark.pov_modes.clone(), + )) } - }); + } + }); // Convert `Vec` to `String` for better readability. let benchmarks_to_run: Vec<_> = benchmarks_to_run .into_iter() @@ -558,6 +554,16 @@ impl PalletCmd { Ok(benchmarks_to_run) } + /// Whether this pallet should be run. + fn pallet_selected(&self, pallet: &Vec) -> bool { + let include = self.pallet.clone().unwrap_or_default(); + + let included = include.is_empty() || include == "*" || include.as_bytes() == pallet; + let excluded = self.exclude_pallets.iter().any(|p| p.as_bytes() == pallet); + + included && !excluded + } + /// Build the genesis storage by either the Genesis Builder API, chain spec or nothing. /// /// Behaviour can be controlled by the `--genesis-builder` flag. @@ -566,12 +572,11 @@ impl PalletCmd { chain_spec: &Option>, ) -> Result { Ok(match (self.genesis_builder, self.runtime.as_ref()) { - (Some(GenesisBuilderPolicy::None), Some(_)) => return Err("Cannot use `--genesis-builder=none` with `--runtime` since the runtime would be ignored.".into()), - (Some(GenesisBuilderPolicy::None), None) => Storage::default(), + (Some(GenesisBuilderPolicy::None), _) => Storage::default(), (Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::Spec), Some(_)) => return Err("Cannot use `--genesis-builder=spec-genesis` with `--runtime` since the runtime would be ignored.".into()), (Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::Spec), None) | (None, None) => { - log::warn!("{WARN_SPEC_GENESIS_CTOR}"); + log::warn!(target: LOG_TARGET, "{WARN_SPEC_GENESIS_CTOR}"); let Some(chain_spec) = chain_spec else { return Err("No chain spec specified to generate the genesis state".into()); }; @@ -593,7 +598,7 @@ impl PalletCmd { }, (Some(GenesisBuilderPolicy::Runtime), None) => return Err("Cannot use `--genesis-builder=runtime` without `--runtime`".into()), (Some(GenesisBuilderPolicy::Runtime), Some(runtime)) | (None, Some(runtime)) => { - log::info!("Loading WASM from {}", runtime.display()); + log::info!(target: LOG_TARGET, "Loading WASM from {}", runtime.display()); let code = fs::read(&runtime).map_err(|e| { format!( @@ -613,7 +618,7 @@ impl PalletCmd { &self, chain_spec: &dyn ChainSpec, ) -> Result { - log::info!("Building genesis state from chain spec runtime"); + log::info!(target: LOG_TARGET, "Building genesis state from chain spec runtime"); let storage = chain_spec .build_storage() .map_err(|e| format!("{ERROR_CANNOT_BUILD_GENESIS}\nError: {e}"))?; @@ -636,8 +641,9 @@ impl PalletCmd { genesis_config_caller.get_storage_for_named_preset(preset).inspect_err(|e| { let presets = genesis_config_caller.preset_names().unwrap_or_default(); log::error!( + target: LOG_TARGET, "Please pick one of the available presets with \ - `--genesis-builder-preset=`. Available presets ({}): {:?}. Error: {:?}", + `--genesis-builder-preset=` or use a different `--genesis-builder-policy`. Available presets ({}): {:?}. Error: {:?}", presets.len(), presets, e @@ -690,10 +696,25 @@ impl PalletCmd { &self, state: &'a BenchmarkingState, ) -> Result, H>> { - log::info!("Loading WASM from state"); - let state = sp_state_machine::backend::BackendRuntimeCode::new(state); + if let Some(runtime) = self.runtime.as_ref() { + log::info!(target: LOG_TARGET, "Loading WASM from file"); + let code = fs::read(runtime).map_err(|e| { + format!( + "Could not load runtime file from path: {}, error: {}", + runtime.display(), + e + ) + })?; + let hash = sp_core::blake2_256(&code).to_vec(); + let wrapped_code = WrappedRuntimeCode(Cow::Owned(code)); + + Ok(FetchedCode::FromFile { wrapped_code, heap_pages: self.heap_pages, hash }) + } else { + log::info!(target: LOG_TARGET, "Loading WASM from state"); + let state = sp_state_machine::backend::BackendRuntimeCode::new(state); - Ok(FetchedCode { state }) + Ok(FetchedCode::FromGenesis { state }) + } } /// Allocation strategy for pallet benchmarking. diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs index a507c1d1f548..412a1a86cb8e 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -53,6 +53,10 @@ pub struct PalletCmd { #[arg(short, long, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))] pub extrinsic: Option, + /// Comma separated list of pallets that should be excluded from the benchmark. + #[arg(long, value_parser, num_args = 1.., value_delimiter = ',')] + pub exclude_pallets: Vec, + /// Run benchmarks for all pallets and extrinsics. /// /// This is equivalent to running `--pallet * --extrinsic *`. diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs index ce37be455e8a..a4799dc92369 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs @@ -18,7 +18,7 @@ //! Various types used by this crate. use sc_cli::Result; -use sp_core::traits::RuntimeCode; +use sp_core::traits::{RuntimeCode, WrappedRuntimeCode}; use sp_runtime::traits::Hash; /// How the genesis state for benchmarking should be build. @@ -43,8 +43,9 @@ pub enum GenesisBuilderPolicy { /// A runtime blob that was either fetched from genesis storage or loaded from a file. // NOTE: This enum is only needed for the annoying lifetime bounds on `RuntimeCode`. Otherwise we // could just directly return the blob. -pub struct FetchedCode<'a, B, H> { - pub state: sp_state_machine::backend::BackendRuntimeCode<'a, B, H>, +pub enum FetchedCode<'a, B, H> { + FromGenesis { state: sp_state_machine::backend::BackendRuntimeCode<'a, B, H> }, + FromFile { wrapped_code: WrappedRuntimeCode<'a>, heap_pages: Option, hash: Vec }, } impl<'a, B, H> FetchedCode<'a, B, H> @@ -54,7 +55,14 @@ where { /// The runtime blob. pub fn code(&'a self) -> Result> { - self.state.runtime_code().map_err(Into::into) + match self { + Self::FromGenesis { state } => state.runtime_code().map_err(Into::into), + Self::FromFile { wrapped_code, heap_pages, hash } => Ok(RuntimeCode { + code_fetcher: wrapped_code, + heap_pages: *heap_pages, + hash: hash.clone(), + }), + } } } diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs index 34f31734da67..28918dd4e6a3 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs @@ -484,9 +484,6 @@ pub(crate) fn write_results( benchmarks: results.clone(), }; - let file_path = fs::canonicalize(&file_path).map_err(|e| { - format!("Could not get absolute path for: {:?}. Error: {:?}", &file_path, e) - })?; let mut output_file = fs::File::create(&file_path).map_err(|e| { format!("Could not write weight file to: {:?}. Error: {:?}", &file_path, e) })?;