From 8f1f4b9a106c8182958ffb55f36b62c4e2023b8b Mon Sep 17 00:00:00 2001 From: Mak Date: Thu, 25 Jul 2024 15:48:42 +0100 Subject: [PATCH 1/2] cmd actions for runtimes (bench & fmt) --- .github/env | 1 + .github/scripts/cmd/_help.py | 26 +++ .github/scripts/cmd/cmd.py | 188 ++++++++++++++++ .github/scripts/cmd/file_header.txt | 15 ++ .github/workflows/cmd.yml | 323 ++++++++++++++++++++++++++++ .github/workflows/fmt.yml | 7 +- .gitignore | 1 + docs/weight-generation.md | 114 +++------- 8 files changed, 588 insertions(+), 87 deletions(-) create mode 100644 .github/scripts/cmd/_help.py create mode 100755 .github/scripts/cmd/cmd.py create mode 100644 .github/scripts/cmd/file_header.txt create mode 100644 .github/workflows/cmd.yml diff --git a/.github/env b/.github/env index 17e6bea555..6ff0ec8a9b 100644 --- a/.github/env +++ b/.github/env @@ -1,2 +1,3 @@ RUST_STABLE_VERSION=1.77.0 RUST_NIGHTLY_VERSION=2024-04-14 +TAPLO_VERSION=0.8.1 diff --git a/.github/scripts/cmd/_help.py b/.github/scripts/cmd/_help.py new file mode 100644 index 0000000000..8ad49dad84 --- /dev/null +++ b/.github/scripts/cmd/_help.py @@ -0,0 +1,26 @@ +import argparse + +""" + +Custom help action for argparse, it prints the help message for the main parser and all subparsers. + +""" + + +class _HelpAction(argparse._HelpAction): + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + + # retrieve subparsers from parser + subparsers_actions = [ + action for action in parser._actions + if isinstance(action, argparse._SubParsersAction)] + # there will probably only be one subparser_action, + # but better save than sorry + for subparsers_action in subparsers_actions: + # get all subparsers and print help + for choice, subparser in subparsers_action.choices.items(): + print("\n### Command '{}'".format(choice)) + print(subparser.format_help()) + + parser.exit() diff --git a/.github/scripts/cmd/cmd.py b/.github/scripts/cmd/cmd.py new file mode 100755 index 0000000000..3d6a628e02 --- /dev/null +++ b/.github/scripts/cmd/cmd.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +import os +import sys +import json +import argparse +import tempfile +import _help + +_HelpAction = _help._HelpAction + +f = open('.github/workflows/runtimes-matrix.json', 'r') +runtimesMatrix = json.load(f) + +runtimeNames = list(map(lambda x: x['name'], runtimesMatrix)) + +common_args = { + '--continue-on-fail': {"action": "store_true", "help": "Won't exit(1) on failed command and continue with next " + "steps. Helpful when you want to push at least successful " + "pallets, and then run failed ones separately"}, + '--quiet': {"action": "store_true", "help": "Won't print start/end/failed messages in Pull Request"}, + '--clean': {"action": "store_true", "help": "Clean up the previous bot's & author's comments in Pull Request " + "which triggered /cmd"}, +} + +parser = argparse.ArgumentParser(prog="/cmd ", description='A command runner for polkadot runtimes repo', add_help=False) +parser.add_argument('--help', action=_HelpAction, help='help for help if you need some help') # help for help + +subparsers = parser.add_subparsers(help='a command to run', dest='command') + +""" +BENCH +""" + +bench_example = '''**Examples**: + + > runs all benchmarks + + %(prog)s + + > runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets + > --quiet makes it to output nothing to PR but reactions + + %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet + + > runs bench for all pallets for polkadot runtime and continues even if some benchmarks fail + + %(prog)s --runtime polkadot --continue-on-fail + + > does not output anything and cleans up the previous bot's & author command triggering comments in PR + + %(prog)s --runtime polkadot kusama --pallet pallet_balances pallet_multisig --quiet --clean + + ''' + +parser_bench = subparsers.add_parser('bench', help='Runs benchmarks', epilog=bench_example, formatter_class=argparse.RawDescriptionHelpFormatter) + +for arg, config in common_args.items(): + parser_bench.add_argument(arg, **config) + +parser_bench.add_argument('--runtime', help='Runtime(s) space separated', choices=runtimeNames, nargs='*', default=runtimeNames) +parser_bench.add_argument('--pallet', help='Pallet(s) space separated', nargs='*', default=[]) + +""" +FMT +""" +parser_fmt = subparsers.add_parser('fmt', help='Formats code') +for arg, config in common_args.items(): + parser_fmt.add_argument(arg, **config) + +args, unknown = parser.parse_known_args() + +print(f'args: {args}') + +if args.command == 'bench': + tempdir = tempfile.TemporaryDirectory() + print(f'Created temp dir: {tempdir.name}') + runtime_pallets_map = {} + failed_benchmarks = {} + successful_benchmarks = {} + + profile = "release" + + print(f'Provided runtimes: {args.runtime}') + # convert to mapped dict + runtimesMatrix = list(filter(lambda x: x['name'] in args.runtime, runtimesMatrix)) + runtimesMatrix = {x['name']: x for x in runtimesMatrix} + print(f'Filtered out runtimes: {runtimesMatrix}') + + # loop over remaining runtimes to collect available pallets + for runtime in runtimesMatrix.values(): + os.system(f"cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks") + print(f'-- listing pallets for benchmark for {runtime["name"]}') + wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm" + output = os.popen( + f"frame-omni-bencher v1 benchmark pallet --no-csv-header --all --list --runtime={wasm_file}").read() + raw_pallets = output.split('\n') + + all_pallets = set() + for pallet in raw_pallets: + if pallet: + all_pallets.add(pallet.split(',')[0]) + + pallets = list(all_pallets) + print(f'Pallets in {runtime}: {pallets}') + runtime_pallets_map[runtime['name']] = pallets + + # filter out only the specified pallets from collected runtimes/pallets + if args.pallet: + print(f'Pallet: {args.pallet}') + new_pallets_map = {} + # keep only specified pallets if they exist in the runtime + for runtime in runtime_pallets_map: + if set(args.pallet).issubset(set(runtime_pallets_map[runtime])): + new_pallets_map[runtime] = args.pallet + + runtime_pallets_map = new_pallets_map + + print(f'Filtered out runtimes & pallets: {runtime_pallets_map}') + + if not runtime_pallets_map: + if args.pallet and not args.runtime: + print(f"No pallets [{args.pallet}] found in any runtime") + elif args.runtime and not args.pallet: + print(f"{args.runtime} runtime does not have any pallets") + elif args.runtime and args.pallet: + print(f"No pallets [{args.pallet}] found in {args.runtime}") + else: + print('No runtimes found') + sys.exit(0) + + header_path = os.path.abspath('./.github/scripts/cmd/file_header.txt') + + for runtime in runtime_pallets_map: + for pallet in runtime_pallets_map[runtime]: + config = runtimesMatrix[runtime] + print(f'-- config: {config}') + default_path = f"./{config['path']}/src/weights" + xcm_path = f"./{config['path']}/src/weights/xcm" + output_path = default_path if not pallet.startswith("pallet_xcm_benchmarks") else xcm_path + print(f'-- benchmarking {pallet} in {runtime} into {output_path}') + + status = os.system(f"frame-omni-bencher v1 benchmark pallet " + f"--extrinsic=* " + f"--runtime=target/{profile}/wbuild/{config['package']}/{config['package'].replace('-', '_')}.wasm " + f"--pallet={pallet} " + f"--header={header_path} " + f"--output={output_path} " + f"--wasm-execution=compiled " + f"--steps=50 " + f"--repeat=20 " + f"--heap-pages=4096 " + ) + if status != 0 and not args.continue_on_fail: + print(f'Failed to benchmark {pallet} in {runtime}') + sys.exit(1) + + # Otherwise collect failed benchmarks and print them at the end + # push failed pallets to failed_benchmarks + if status != 0: + failed_benchmarks[f'{runtime}'] = failed_benchmarks.get(f'{runtime}', []) + [pallet] + else: + successful_benchmarks[f'{runtime}'] = successful_benchmarks.get(f'{runtime}', []) + [pallet] + + if failed_benchmarks: + print('❌ Failed benchmarks of runtimes/pallets:') + for runtime, pallets in failed_benchmarks.items(): + print(f'-- {runtime}: {pallets}') + + if successful_benchmarks: + print('βœ… Successful benchmarks of runtimes/pallets:') + for runtime, pallets in successful_benchmarks.items(): + print(f'-- {runtime}: {pallets}') + + tempdir.cleanup() + +elif args.command == 'fmt': + nightly_version = os.getenv('RUST_NIGHTLY_VERSION') + command = f"cargo +nightly-{nightly_version} fmt"; + print('Formatting with `{command}`') + nightly_status = os.system(f'{command}') + taplo_status = os.system('taplo format --config .config/taplo.toml') + + if (nightly_status != 0 or taplo_status != 0) and not args.continue_on_fail: + print('❌ Failed to format code') + sys.exit(1) + +print('πŸš€ Done') diff --git a/.github/scripts/cmd/file_header.txt b/.github/scripts/cmd/file_header.txt new file mode 100644 index 0000000000..9fc718fdc0 --- /dev/null +++ b/.github/scripts/cmd/file_header.txt @@ -0,0 +1,15 @@ +// Copyright (C) Parity Technologies and the various Polkadot contributors, see Contributions.md +// for a list of specific contributors. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. diff --git a/.github/workflows/cmd.yml b/.github/workflows/cmd.yml new file mode 100644 index 0000000000..380cf5f1be --- /dev/null +++ b/.github/workflows/cmd.yml @@ -0,0 +1,323 @@ +name: Command + +on: + issue_comment: # listen for comments on issues + types: [created] + +permissions: # allow the action to comment on the PR + contents: write + issues: write + pull-requests: write + actions: read + +jobs: + acknowledge: + if: ${{ startsWith(github.event.comment.body, '/cmd') }} + runs-on: ubuntu-latest + steps: + - name: Add reaction to triggered comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'eyes' + }) + + clean: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Clean previous comments + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--clean') }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }).then(comments => { + for (let comment of comments.data) { + console.log(comment) + if ( + ${{ github.event.comment.id }} !== comment.id && + ( + ((comment.body.startsWith('Command') || comment.body.startsWith('
Command')) && comment.user.type === 'Bot') || + (comment.body.startsWith('/cmd') && comment.user.login === context.actor) + ) + ) { + github.rest.issues.deleteComment({ + comment_id: comment.id, + owner: context.repo.owner, + repo: context.repo.repo + }) + } + } + }) + help: + needs: clean + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--help') }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get command + uses: actions-ecosystem/action-regex-match@v2 + id: get-pr-comment + with: + text: ${{ github.event.comment.body }} + regex: '^(\/cmd )(.*)$' + + - name: Save output of help + id: help + env: + CMD: ${{ steps.get-pr-comment.outputs.group2 }} # to avoid "" around the command + run: | + echo 'help<> $GITHUB_OUTPUT + python3 .github/scripts/cmd/cmd.py $CMD >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Comment PR (Help) + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `
Command help:${{ steps.help.outputs.help }}
` + }) + + - name: Add confused reaction on failure + uses: actions/github-script@v7 + if: ${{ failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Add πŸ‘ reaction on success + uses: actions/github-script@v7 + if: ${{ !failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: '+1' + }) + + cmd: + needs: clean + env: + JOB_NAME: 'cmd' + if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(github.event.comment.body, '--help') }} + # TODO: replace runner for bench, when it's ready + runs-on: ${{ startsWith(github.event.comment.body, '/cmd bench') && 'ubuntu-22.04' || 'ubuntu-22.04' }} + steps: + - name: Install updates and protobuf-compiler + if: startsWith(github.event.comment.body, '/cmd bench') + run: | + sudo apt update && sudo apt install --assume-yes \ + openssl pkg-config g++ make cmake protobuf-compiler curl libssl-dev libclang-dev libudev-dev git jq + + - name: Get command + uses: actions-ecosystem/action-regex-match@v2 + id: get-pr-comment + with: + text: ${{ github.event.comment.body }} + regex: '^(\/cmd )(.*)$' + + - name: Build workflow link + if: ${{ !contains(github.event.comment.body, '--quiet') }} + id: build-link + run: | + # Get exactly the CMD job link, filtering out the other jobs + jobLink=$(curl -s \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs | jq '.jobs[] | select(.name | contains("${{ env.JOB_NAME }}")) | .html_url') + + runLink=$(curl -s \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }} | jq '.html_url') + + echo "job_url=${jobLink}" + echo "run_url=${runLink}" + echo "job_url=$jobLink" >> $GITHUB_OUTPUT + echo "run_url=$runLink" >> $GITHUB_OUTPUT + + + - name: Comment PR (Start) + if: ${{ !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let job_url = ${{ steps.build-link.outputs.job_url }} + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has started πŸš€ [See logs here](${job_url})` + }) + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Set rust version via common env file + run: cat .github/env >> $GITHUB_ENV + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + targets: "wasm32-unknown-unknown,x86_64-unknown-linux-musl" + components: "rust-src rustfmt clippy" + toolchain: "nightly-${{env.RUST_NIGHTLY_VERSION}}" + + - name: Install dependencies for bench + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + run: cargo install subweight frame-omni-bencher --locked + + - name: Install dependencies for fmt + if: startsWith(steps.get-pr-comment.outputs.group2, 'fmt') + run: cargo install taplo-cli --version ${{ env.TAPLO_VERSION }} + + - name: Setup Cache + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2.7.0 + with: + shared-key: "fellowship-cmd" + + - name: Run cmd + id: cmd + env: + CMD: ${{ steps.get-pr-comment.outputs.group2 }} # to avoid "" around the command + run: | + echo "Running command: $CMD" + git remote -v + # TODO: uncomment + python3 .github/scripts/cmd/cmd.py $CMD + git status + git diff + + - name: Commit changes + run: | + if [ -n "$(git status --porcelain)" ]; then + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + git pull origin ${{ github.head_ref }} + git add . + git restore --staged Cargo.lock # ignore changes in Cargo.lock + git commit -m "Update from ${{ github.actor }} running command '${{ steps.get-pr-comment.outputs.group2 }}'" || true + git push origin ${{ github.head_ref }} + else + echo "Nothing to commit"; + fi + + - name: Run Subweight + id: subweight + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + shell: bash + run: | + git fetch + result=$(subweight compare commits \ + --path-pattern "./**/weights/**/*.rs" \ + --method asymptotic \ + --format markdown \ + --no-color \ + --change added changed \ + --ignore-errors \ + refs/remotes/origin/main ${{ github.ref }}) + + # Save the multiline result to the output + { + echo "result<> $GITHUB_OUTPUT + + - name: Comment PR (End) + if: ${{ !failure() && !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + env: + SUBWEIGHT: '${{ steps.subweight.outputs.result }}' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let runUrl = ${{ steps.build-link.outputs.run_url }} + let subweight = process.env.SUBWEIGHT; + + let subweightCollapsed = subweight + ? `
\n\nSubweight results:\n\n${subweight}\n\n
` + : ''; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has finished βœ… [See logs here](${runUrl})${subweightCollapsed}` + }) + + - name: Comment PR (Failure) + if: ${{ failure() && !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let jobUrl = ${{ steps.build-link.outputs.job_url }} + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has failed ❌! [See logs here](${jobUrl})` + }) + + - name: Add πŸ˜• reaction on failure + uses: actions/github-script@v7 + if: ${{ failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Add πŸ‘ reaction on success + uses: actions/github-script@v7 + if: ${{ !failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: '+1' + }) + + diff --git a/.github/workflows/fmt.yml b/.github/workflows/fmt.yml index 11d4df7372..2b115bd863 100644 --- a/.github/workflows/fmt.yml +++ b/.github/workflows/fmt.yml @@ -24,7 +24,12 @@ jobs: run: cat .github/env >> $GITHUB_ENV - name: Install nightly toolchain - run: rustup toolchain install "nightly-$RUST_NIGHTLY_VERSION" --profile minimal --component rustfmt + run: | + rustup toolchain install "nightly-$RUST_NIGHTLY_VERSION" --profile minimal --component rustfmt + cargo install taplo-cli --version $TAPLO_VERSION - name: Rustfmt (check) run: cargo +nightly-$RUST_NIGHTLY_VERSION fmt --all -- --check + + - name: Taplo (check) + run: taplo format --check --config .config/taplo.toml diff --git a/.gitignore b/.gitignore index 4c9006e4e5..d591ea0a06 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ rls*.log runtime/wasm/target/ substrate.code-workspace target/ +**/__pycache__/ diff --git a/docs/weight-generation.md b/docs/weight-generation.md index 6904db0ca3..fea2d41f5f 100644 --- a/docs/weight-generation.md +++ b/docs/weight-generation.md @@ -1,101 +1,43 @@ # Weight Generation -To generate weights for a runtime +To generate weights for a runtime. +Weights generation is using self-hosted runner which is provided by [Amforc](https://amforc.com/), the rest commands are using standard Github runners on `ubuntu-latest` or `ubuntu-20.04`. +Self-hosted runner for benchmarks is configured to meet requirements of reference hardware for running validators https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware -1. Build `chain-spec-generator` with `--profile production --features runtime-benchmarks` -2. Use it to build a chain spec for your runtime, e.g. `./target/production/chain-spec-generator --raw polkadot-local > polkadot-chain-spec.json` -3. Create `file_header.txt` +In a PR run the actions through comment: -```text -// Copyright (C) Parity Technologies and the various Polkadot contributors, see Contributions.md -// for a list of specific contributors. -// SPDX-License-Identifier: Apache-2.0 +```sh +/cmd bench --help # outputs the actual usage documentation with examples and supported runtimes -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -``` - -4. `rsync` chain spec/s and the file header to a benchmark machine - -5. Build `polkadot` binary from the latest release of `polkadot-sdk` with `--profile production --features runtime-benchmarks --bin polkadot` on the benchmark machine - -6. Run on the benchmark machine: - -```bash -#!/bin/bash - -# Default value is 'polkadot', but you can override it by passing a different value as an argument -CHAIN=${1:-polkadot} - -pallets=($( - ./target/production/polkadot benchmark pallet --list \ - --chain=./$CHAIN-chain-spec.json | - tail -n+2 | - cut -d',' -f1 | - sort | - uniq -)); +# or -mkdir -p ./$CHAIN-weights -for pallet in "${pallets[@]}"; do - output_file=./$CHAIN-weights/ - # a little hack for pallet_xcm_benchmarks - we want to output them to a nested directory - if [[ "$pallet" == "pallet_xcm_benchmarks::generic" ]] || [[ "$pallet" == "pallet_xcm_benchmarks::fungible" ]]; then - mkdir -p ./$CHAIN-weights/xcm - output_file="${output_file}xcm/${pallet//::/_}.rs" - fi - echo "Running benchmarks for $pallet to $output_file" - ./target/production/polkadot benchmark pallet \ - --chain=./$CHAIN-chain-spec.json \ - --steps=50 \ - --repeat=20 \ - --pallet=$pallet \ - --extrinsic=* \ - --wasm-execution=compiled \ - --heap-pages=4096 \ - --output="$output_file" \ - --header=./file_header.txt -done +/cmd --help # to see all available commands ``` -You probably want to do this inside a `tmux` session or something similar (e.g., `nohup &`), as it will take a while (several hours). - -7. `rsync` the weights back to your local machine, replacing the existing weights. - -8. Manually fix XCM weights by -- Replacing `impl xxx::yyy::WeightInfo for WeightInfo {` with `impl WeightInfo {` -- Marking all functions `pub(crate)` -- Removing any unused functions - -9. Commit the weight changes. +To generate weights for all pallets in a particular runtime(s), run the following command: +```sh +/cmd bench --runtime kusama polkadot +``` -10. Ensure the changes are reasonable. If not installed, `cargo install subweight`, check the weight changes: - ``` - subweight compare commits \ - --path-pattern "./**/weights/**/*.rs" \ - --method asymptotic \ - --ignore-errors \ - \ - ` - ``` - _Hint1: Add `--format markdown --no-color` for markdown-compatible results._ +> **πŸ“ Note**: The action is not being run right-away, it will be queued and run in the next available runner. So might be quick, but might also take up to 10 mins (That's in control of Github). +Once the action is run, you'll see reaction πŸ‘€ on original comment, and if you didn't pass `--quiet` - it will also send a link to a pipeline when started, and link to whole workflow when finished. - _Hint2: Change `--path-pattern "./**/weights/**/*.rs"` to e.g. `--path-pattern "./relay/polkadot/weights/**/*.rs"` for a specific runtime._ +> **πŸ’‘Hint #1** : if you run all runtimes or all pallets, it might be that some pallet in the middle is failed to generate weights, thus it stops (fails) the whole pipeline. +> If you want, you can make it to continue running, even if some pallets are failed, add `--continue-on-fail` flag to the command. The report will include which runtimes/pallets have failed, then you can re-run them separately after all is done. - _Hint3: Add `--change added changed` to include only relevant changes._ +This way it runs all possible runtimes for the specified pallets, if it finds them in the runtime +```sh +/cmd bench --pallet pallet_balances pallet_xcm_benchmarks::generic pallet_xcm_benchmarks::fungible +``` -## FAQ +If you want to run all specific pallet(s) for specific runtime(s), you can do it like this: +```sh +/cmd bench --runtime bridge-hub-polkadot --pallet pallet_xcm_benchmarks::generic pallet_xcm_benchmarks::fungible +``` -### What benchmark machine spec should I use? -See the [Polkadot Wiki Reference Hardware](https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#standard-hardware). +> **πŸ’‘Hint #2** : Sometimes when you run too many commands, or they keep failing and you're rerunning them again, it's handy to add `--clean` flag to the command. This will clean up all yours and bot's comments in PR relevant to /cmd commands. +```sh +/cmd bench --runtime kusama polkadot --pallet=pallet_balances --clean --continue-on-fail +``` From d530cea85f94fee11ea5b8a9202b26967c479756 Mon Sep 17 00:00:00 2001 From: Maksym H <1177472+mordamax@users.noreply.github.com> Date: Wed, 31 Jul 2024 23:40:23 +0100 Subject: [PATCH 2/2] add trim() to pallet name Co-authored-by: Oliver Tale-Yazdi --- .github/scripts/cmd/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/cmd/cmd.py b/.github/scripts/cmd/cmd.py index 3d6a628e02..52b1e9dc85 100755 --- a/.github/scripts/cmd/cmd.py +++ b/.github/scripts/cmd/cmd.py @@ -99,7 +99,7 @@ all_pallets = set() for pallet in raw_pallets: if pallet: - all_pallets.add(pallet.split(',')[0]) + all_pallets.add(pallet.split(',')[0].trim()) pallets = list(all_pallets) print(f'Pallets in {runtime}: {pallets}')