From 16d2db78d26d70d4ead015d3428d0292401c30b4 Mon Sep 17 00:00:00 2001 From: Maksym H <1177472+mordamax@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:56:15 +0100 Subject: [PATCH] cmd actions bench & fmt (#406) Closes https://github.com/polkadot-fellows/runtimes/issues/128 Closes https://github.com/paritytech/infrastructure/issues/41 Adds ability to run some standard helper commands, for now only for benchmarks and fmt (with taplo) It will commit the result of operation back into PR with some reports and/or links to logs bench uses `gitrun-001` runner which corresponds to [Validator Reference Hardware](https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#standard-hardware) - [x] Does not require a CHANGELOG entry
Examples (screenshots): ### Benchmarks ![Google Chrome 2024-07-26 13 33 05](https://github.com/user-attachments/assets/d1d32f7e-295a-4f52-bb69-ff93a4a61493) ### Bench results with Subweight report ![Google Chrome 2024-07-26 13 35 36](https://github.com/user-attachments/assets/e907b8f2-f818-475e-9d52-fdf48a0455f8) ### /cmd [command] --clean - removing bot's and author's command runs to clean up PR ![Google Chrome 2024-07-26 13 33 56](https://github.com/user-attachments/assets/6e21f616-b4c1-4ca5-a89a-48e64a46c625) ### /cmd [command] --help - outputs the usage instructions ![Google Chrome 2024-07-26 13 33 25](https://github.com/user-attachments/assets/73d46493-4ab1-4f9d-a5e8-094afc4f2a49)
--------- Co-authored-by: Oliver Tale-Yazdi --- .github/env | 1 + .github/scripts/cmd/_help.py | 26 ++ .github/scripts/cmd/cmd.py | 188 +++++++++ .github/scripts/cmd/file_header.txt | 15 + .github/workflows/cmd.yml | 364 ++++++++++++++++++ .github/workflows/fmt.yml | 7 +- .gitignore | 1 + docs/running-commands.md | 32 ++ docs/weight-generation.md | 114 ++---- relay/common/Cargo.toml | 8 +- relay/kusama/Cargo.toml | 4 +- relay/polkadot/Cargo.toml | 8 +- .../coretime/coretime-polkadot/Cargo.toml | 2 +- system-parachains/encointer/Cargo.toml | 4 +- 14 files changed, 674 insertions(+), 100 deletions(-) create mode 100644 .github/scripts/cmd/_help.py create mode 100755 .github/scripts/cmd/cmd.py create mode 100644 .github/scripts/cmd/file_header.txt create mode 100644 .github/workflows/cmd.yml create mode 100644 docs/running-commands.md diff --git a/.github/env b/.github/env index 17e6bea555..6ff0ec8a9b 100644 --- a/.github/env +++ b/.github/env @@ -1,2 +1,3 @@ RUST_STABLE_VERSION=1.77.0 RUST_NIGHTLY_VERSION=2024-04-14 +TAPLO_VERSION=0.8.1 diff --git a/.github/scripts/cmd/_help.py b/.github/scripts/cmd/_help.py new file mode 100644 index 0000000000..8ad49dad84 --- /dev/null +++ b/.github/scripts/cmd/_help.py @@ -0,0 +1,26 @@ +import argparse + +""" + +Custom help action for argparse, it prints the help message for the main parser and all subparsers. + +""" + + +class _HelpAction(argparse._HelpAction): + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + + # retrieve subparsers from parser + subparsers_actions = [ + action for action in parser._actions + if isinstance(action, argparse._SubParsersAction)] + # there will probably only be one subparser_action, + # but better save than sorry + for subparsers_action in subparsers_actions: + # get all subparsers and print help + for choice, subparser in subparsers_action.choices.items(): + print("\n### Command '{}'".format(choice)) + print(subparser.format_help()) + + parser.exit() diff --git a/.github/scripts/cmd/cmd.py b/.github/scripts/cmd/cmd.py new file mode 100755 index 0000000000..589ce7e173 --- /dev/null +++ b/.github/scripts/cmd/cmd.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +import os +import sys +import json +import argparse +import tempfile +import _help + +_HelpAction = _help._HelpAction + +f = open('.github/workflows/runtimes-matrix.json', 'r') +runtimesMatrix = json.load(f) + +runtimeNames = list(map(lambda x: x['name'], runtimesMatrix)) + +common_args = { + '--continue-on-fail': {"action": "store_true", "help": "Won't exit(1) on failed command and continue with next " + "steps. Helpful when you want to push at least successful " + "pallets, and then run failed ones separately"}, + '--quiet': {"action": "store_true", "help": "Won't print start/end/failed messages in Pull Request"}, + '--clean': {"action": "store_true", "help": "Clean up the previous bot's & author's comments in Pull Request " + "which triggered /cmd"}, +} + +parser = argparse.ArgumentParser(prog="/cmd ", description='A command runner for polkadot runtimes repo', add_help=False) +parser.add_argument('--help', action=_HelpAction, help='help for help if you need some help') # help for help + +subparsers = parser.add_subparsers(help='a command to run', dest='command') + +""" +BENCH +""" + +bench_example = '''**Examples**: + + > runs all benchmarks + + %(prog)s + + > runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets + > --quiet makes it to output nothing to PR but reactions + + %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet + + > runs bench for all pallets for polkadot runtime and continues even if some benchmarks fail + + %(prog)s --runtime polkadot --continue-on-fail + + > does not output anything and cleans up the previous bot's & author command triggering comments in PR + + %(prog)s --runtime polkadot kusama --pallet pallet_balances pallet_multisig --quiet --clean + + ''' + +parser_bench = subparsers.add_parser('bench', help='Runs benchmarks', epilog=bench_example, formatter_class=argparse.RawDescriptionHelpFormatter) + +for arg, config in common_args.items(): + parser_bench.add_argument(arg, **config) + +parser_bench.add_argument('--runtime', help='Runtime(s) space separated', choices=runtimeNames, nargs='*', default=runtimeNames) +parser_bench.add_argument('--pallet', help='Pallet(s) space separated', nargs='*', default=[]) + +""" +FMT +""" +parser_fmt = subparsers.add_parser('fmt', help='Formats code') +for arg, config in common_args.items(): + parser_fmt.add_argument(arg, **config) + +args, unknown = parser.parse_known_args() + +print(f'args: {args}') + +if args.command == 'bench': + tempdir = tempfile.TemporaryDirectory() + print(f'Created temp dir: {tempdir.name}') + runtime_pallets_map = {} + failed_benchmarks = {} + successful_benchmarks = {} + + profile = "release" + + print(f'Provided runtimes: {args.runtime}') + # convert to mapped dict + runtimesMatrix = list(filter(lambda x: x['name'] in args.runtime, runtimesMatrix)) + runtimesMatrix = {x['name']: x for x in runtimesMatrix} + print(f'Filtered out runtimes: {runtimesMatrix}') + + # loop over remaining runtimes to collect available pallets + for runtime in runtimesMatrix.values(): + os.system(f"cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks") + print(f'-- listing pallets for benchmark for {runtime["name"]}') + wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm" + output = os.popen( + f"frame-omni-bencher v1 benchmark pallet --no-csv-header --all --list --runtime={wasm_file}").read() + raw_pallets = output.split('\n') + + all_pallets = set() + for pallet in raw_pallets: + if pallet: + all_pallets.add(pallet.split(',')[0].strip()) + + pallets = list(all_pallets) + print(f'Pallets in {runtime}: {pallets}') + runtime_pallets_map[runtime['name']] = pallets + + # filter out only the specified pallets from collected runtimes/pallets + if args.pallet: + print(f'Pallet: {args.pallet}') + new_pallets_map = {} + # keep only specified pallets if they exist in the runtime + for runtime in runtime_pallets_map: + if set(args.pallet).issubset(set(runtime_pallets_map[runtime])): + new_pallets_map[runtime] = args.pallet + + runtime_pallets_map = new_pallets_map + + print(f'Filtered out runtimes & pallets: {runtime_pallets_map}') + + if not runtime_pallets_map: + if args.pallet and not args.runtime: + print(f"No pallets [{args.pallet}] found in any runtime") + elif args.runtime and not args.pallet: + print(f"{args.runtime} runtime does not have any pallets") + elif args.runtime and args.pallet: + print(f"No pallets [{args.pallet}] found in {args.runtime}") + else: + print('No runtimes found') + sys.exit(0) + + header_path = os.path.abspath('./.github/scripts/cmd/file_header.txt') + + for runtime in runtime_pallets_map: + for pallet in runtime_pallets_map[runtime]: + config = runtimesMatrix[runtime] + print(f'-- config: {config}') + default_path = f"./{config['path']}/src/weights" + xcm_path = f"./{config['path']}/src/weights/xcm" + output_path = default_path if not pallet.startswith("pallet_xcm_benchmarks") else xcm_path + print(f'-- benchmarking {pallet} in {runtime} into {output_path}') + + status = os.system(f"frame-omni-bencher v1 benchmark pallet " + f"--extrinsic=* " + f"--runtime=target/{profile}/wbuild/{config['package']}/{config['package'].replace('-', '_')}.wasm " + f"--pallet={pallet} " + f"--header={header_path} " + f"--output={output_path} " + f"--wasm-execution=compiled " + f"--steps=50 " + f"--repeat=20 " + f"--heap-pages=4096 " + ) + if status != 0 and not args.continue_on_fail: + print(f'Failed to benchmark {pallet} in {runtime}') + sys.exit(1) + + # Otherwise collect failed benchmarks and print them at the end + # push failed pallets to failed_benchmarks + if status != 0: + failed_benchmarks[f'{runtime}'] = failed_benchmarks.get(f'{runtime}', []) + [pallet] + else: + successful_benchmarks[f'{runtime}'] = successful_benchmarks.get(f'{runtime}', []) + [pallet] + + if failed_benchmarks: + print('❌ Failed benchmarks of runtimes/pallets:') + for runtime, pallets in failed_benchmarks.items(): + print(f'-- {runtime}: {pallets}') + + if successful_benchmarks: + print('βœ… Successful benchmarks of runtimes/pallets:') + for runtime, pallets in successful_benchmarks.items(): + print(f'-- {runtime}: {pallets}') + + tempdir.cleanup() + +elif args.command == 'fmt': + nightly_version = os.getenv('RUST_NIGHTLY_VERSION') + command = f"cargo +nightly-{nightly_version} fmt" + print('Formatting with `{command}`') + nightly_status = os.system(f'{command}') + taplo_status = os.system('taplo format --config .config/taplo.toml') + + if (nightly_status != 0 or taplo_status != 0) and not args.continue_on_fail: + print('❌ Failed to format code') + sys.exit(1) + +print('πŸš€ Done') diff --git a/.github/scripts/cmd/file_header.txt b/.github/scripts/cmd/file_header.txt new file mode 100644 index 0000000000..9fc718fdc0 --- /dev/null +++ b/.github/scripts/cmd/file_header.txt @@ -0,0 +1,15 @@ +// Copyright (C) Parity Technologies and the various Polkadot contributors, see Contributions.md +// for a list of specific contributors. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. diff --git a/.github/workflows/cmd.yml b/.github/workflows/cmd.yml new file mode 100644 index 0000000000..10c82e0650 --- /dev/null +++ b/.github/workflows/cmd.yml @@ -0,0 +1,364 @@ +name: Command + +on: + issue_comment: # listen for comments on issues + types: [created] + +permissions: # allow the action to comment on the PR + contents: write + issues: write + pull-requests: write + actions: read + +jobs: + fellows: + runs-on: ubuntu-latest + outputs: + github-handles: ${{ steps.load-fellows.outputs.github-handles }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load fellows + id: load-fellows + uses: paritytech/get-fellows-action@v1.2.0 + + reject-non-fellows: + needs: fellows + if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ubuntu-latest + steps: + - name: Add reaction to rejected comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Comment PR (Rejected) + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Sorry, only fellows can run commands.` + }) + + acknowledge: + needs: fellows + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ubuntu-latest + steps: + - name: Add reaction to triggered comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'eyes' + }) + + clean: + needs: fellows + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Clean previous comments + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--clean') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }).then(comments => { + for (let comment of comments.data) { + console.log(comment) + if ( + ${{ github.event.comment.id }} !== comment.id && + ( + ((comment.body.startsWith('Command') || comment.body.startsWith('
Command')) && comment.user.type === 'Bot') || + (comment.body.startsWith('/cmd') && comment.user.login === context.actor) + ) + ) { + github.rest.issues.deleteComment({ + comment_id: comment.id, + owner: context.repo.owner, + repo: context.repo.repo + }) + } + } + }) + help: + needs: [clean, fellows] + if: ${{ startsWith(github.event.comment.body, '/cmd') && contains(github.event.comment.body, '--help') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get command + uses: actions-ecosystem/action-regex-match@v2 + id: get-pr-comment + with: + text: ${{ github.event.comment.body }} + regex: '^(\/cmd )([\s\w-]+)$' + + - name: Save output of help + id: help + env: + CMD: ${{ steps.get-pr-comment.outputs.group2 }} # to avoid "" around the command + run: | + echo 'help<> $GITHUB_OUTPUT + python3 .github/scripts/cmd/cmd.py $CMD >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Comment PR (Help) + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `
Command help:${{ steps.help.outputs.help }}
` + }) + + - name: Add confused reaction on failure + uses: actions/github-script@v7 + if: ${{ failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Add πŸ‘ reaction on success + uses: actions/github-script@v7 + if: ${{ !failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: '+1' + }) + + cmd: + needs: [clean, fellows] + env: + JOB_NAME: 'cmd' + if: ${{ startsWith(github.event.comment.body, '/cmd') && !contains(github.event.comment.body, '--help') && contains(needs.fellows.outputs.github-handles, github.event.sender.login) }} + runs-on: ${{ startsWith(github.event.comment.body, '/cmd bench') && 'gitrun-001' || 'ubuntu-22.04' }} + steps: + - name: Install updates and protobuf-compiler + if: startsWith(github.event.comment.body, '/cmd bench') + run: | + sudo apt update && sudo apt install --assume-yes \ + openssl pkg-config g++ make cmake protobuf-compiler curl libssl-dev libclang-dev libudev-dev git jq + + - name: Get command + uses: actions-ecosystem/action-regex-match@v2 + id: get-pr-comment + with: + text: ${{ github.event.comment.body }} + regex: '^(\/cmd )([\s\w-]+)$' + + - name: Build workflow link + if: ${{ !contains(github.event.comment.body, '--quiet') }} + id: build-link + run: | + # Get exactly the CMD job link, filtering out the other jobs + jobLink=$(curl -s \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs | jq '.jobs[] | select(.name | contains("${{ env.JOB_NAME }}")) | .html_url') + + runLink=$(curl -s \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }} | jq '.html_url') + + echo "job_url=${jobLink}" + echo "run_url=${runLink}" + echo "job_url=$jobLink" >> $GITHUB_OUTPUT + echo "run_url=$runLink" >> $GITHUB_OUTPUT + + + - name: Comment PR (Start) + if: ${{ !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let job_url = ${{ steps.build-link.outputs.job_url }} + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has started πŸš€ [See logs here](${job_url})` + }) + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Set rust version via common env file + run: cat .github/env >> $GITHUB_ENV + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + targets: "wasm32-unknown-unknown,x86_64-unknown-linux-musl" + components: "rust-src rustfmt clippy" + toolchain: "nightly-${{env.RUST_NIGHTLY_VERSION}}" + + - name: Install dependencies for bench + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + run: cargo install subweight frame-omni-bencher --locked + + - name: Install dependencies for fmt + if: startsWith(steps.get-pr-comment.outputs.group2, 'fmt') + run: cargo install taplo-cli --version ${{ env.TAPLO_VERSION }} + + - name: Setup Cache + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2.7.0 + with: + shared-key: "fellowship-cmd" + + - name: Run cmd + id: cmd + env: + CMD: ${{ steps.get-pr-comment.outputs.group2 }} # to avoid "" around the command + run: | + echo "Running command: $CMD" + git remote -v + python3 .github/scripts/cmd/cmd.py $CMD + git status + git diff + + - name: Commit changes + run: | + if [ -n "$(git status --porcelain)" ]; then + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + git pull origin ${{ github.head_ref }} + git add . + git restore --staged Cargo.lock # ignore changes in Cargo.lock + git commit -m "Update from ${{ github.actor }} running command '${{ steps.get-pr-comment.outputs.group2 }}'" || true + git push origin ${{ github.head_ref }} + else + echo "Nothing to commit"; + fi + + - name: Run Subweight + id: subweight + if: startsWith(steps.get-pr-comment.outputs.group2, 'bench') + shell: bash + run: | + git fetch + result=$(subweight compare commits \ + --path-pattern "./**/weights/**/*.rs" \ + --method asymptotic \ + --format markdown \ + --no-color \ + --change added changed \ + --ignore-errors \ + refs/remotes/origin/main ${{ github.ref }}) + + # Save the multiline result to the output + { + echo "result<> $GITHUB_OUTPUT + + - name: Comment PR (End) + if: ${{ !failure() && !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + env: + SUBWEIGHT: '${{ steps.subweight.outputs.result }}' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let runUrl = ${{ steps.build-link.outputs.run_url }} + let subweight = process.env.SUBWEIGHT; + + let subweightCollapsed = subweight + ? `
\n\nSubweight results:\n\n${subweight}\n\n
` + : ''; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has finished βœ… [See logs here](${runUrl})${subweightCollapsed}` + }) + + - name: Comment PR (Failure) + if: ${{ failure() && !contains(github.event.comment.body, '--quiet') }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let jobUrl = ${{ steps.build-link.outputs.job_url }} + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has failed ❌! [See logs here](${jobUrl})` + }) + + - name: Add πŸ˜• reaction on failure + uses: actions/github-script@v7 + if: ${{ failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: 'confused' + }) + + - name: Add πŸ‘ reaction on success + uses: actions/github-script@v7 + if: ${{ !failure() }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.reactions.createForIssueComment({ + comment_id: ${{ github.event.comment.id }}, + owner: context.repo.owner, + repo: context.repo.repo, + content: '+1' + }) + + diff --git a/.github/workflows/fmt.yml b/.github/workflows/fmt.yml index 11d4df7372..2b115bd863 100644 --- a/.github/workflows/fmt.yml +++ b/.github/workflows/fmt.yml @@ -24,7 +24,12 @@ jobs: run: cat .github/env >> $GITHUB_ENV - name: Install nightly toolchain - run: rustup toolchain install "nightly-$RUST_NIGHTLY_VERSION" --profile minimal --component rustfmt + run: | + rustup toolchain install "nightly-$RUST_NIGHTLY_VERSION" --profile minimal --component rustfmt + cargo install taplo-cli --version $TAPLO_VERSION - name: Rustfmt (check) run: cargo +nightly-$RUST_NIGHTLY_VERSION fmt --all -- --check + + - name: Taplo (check) + run: taplo format --check --config .config/taplo.toml diff --git a/.gitignore b/.gitignore index 4c9006e4e5..d591ea0a06 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ rls*.log runtime/wasm/target/ substrate.code-workspace target/ +**/__pycache__/ diff --git a/docs/running-commands.md b/docs/running-commands.md new file mode 100644 index 0000000000..a7d55e371c --- /dev/null +++ b/docs/running-commands.md @@ -0,0 +1,32 @@ +# Running Commands in PRs + +You can run commands in PRs by triggering it via comment. It will use the context of your PR and post the results back. + +## Usage + +`/cmd --help` to see all available commands and usage format + +`/cmd --help` to see the usage of a specific command + + +### Commands + +- `/cmd fmt` to format the code in the PR. It commits back with the formatted code (fmt) and configs (taplo). + +- `/cmd bench` to generate weights for a runtime. Read more about [Weight Generation](./weight-generation.md) + +### Flags + +1.`--quiet` to suppress the output of the command in the comments. +By default, the Start and End/Failure of the command will be commented with the link to a pipeline. +If you want to avoid, use this flag. Go to [Action Tab](https://github.com/polkadot-fellows/runtimes/actions/workflows/cmd.yml) to see the pipeline status. + +2.`--continue-on-fail` to continue running the command even if something inside a command (like specific pallet weight generation) are failed. +Basically avoids interruption in the middle with `exit 1` +The pipeline logs will include what is failed (like which runtimes/pallets), then you can re-run them separately or not. + +3.`--clean` to clean up all yours and bot's comments in PR relevant to `/cmd` commands. If you run too many commands, or they keep failing, and you're rerunning them again, it's handy to add this flag to keep a PR clean. + +### Adding new Commands +Feel free to add new commands to the workflow, however **_note_** that triggered workflows will use the actions from `main` (default) branch, meaning they will take effect only after the PR with new changes/command is merged. +If you want to test the new command, it's better to test in your fork and local-to-fork PRs, where you control the default branch. diff --git a/docs/weight-generation.md b/docs/weight-generation.md index 6904db0ca3..fea2d41f5f 100644 --- a/docs/weight-generation.md +++ b/docs/weight-generation.md @@ -1,101 +1,43 @@ # Weight Generation -To generate weights for a runtime +To generate weights for a runtime. +Weights generation is using self-hosted runner which is provided by [Amforc](https://amforc.com/), the rest commands are using standard Github runners on `ubuntu-latest` or `ubuntu-20.04`. +Self-hosted runner for benchmarks is configured to meet requirements of reference hardware for running validators https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware -1. Build `chain-spec-generator` with `--profile production --features runtime-benchmarks` -2. Use it to build a chain spec for your runtime, e.g. `./target/production/chain-spec-generator --raw polkadot-local > polkadot-chain-spec.json` -3. Create `file_header.txt` +In a PR run the actions through comment: -```text -// Copyright (C) Parity Technologies and the various Polkadot contributors, see Contributions.md -// for a list of specific contributors. -// SPDX-License-Identifier: Apache-2.0 +```sh +/cmd bench --help # outputs the actual usage documentation with examples and supported runtimes -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -``` - -4. `rsync` chain spec/s and the file header to a benchmark machine - -5. Build `polkadot` binary from the latest release of `polkadot-sdk` with `--profile production --features runtime-benchmarks --bin polkadot` on the benchmark machine - -6. Run on the benchmark machine: - -```bash -#!/bin/bash - -# Default value is 'polkadot', but you can override it by passing a different value as an argument -CHAIN=${1:-polkadot} - -pallets=($( - ./target/production/polkadot benchmark pallet --list \ - --chain=./$CHAIN-chain-spec.json | - tail -n+2 | - cut -d',' -f1 | - sort | - uniq -)); +# or -mkdir -p ./$CHAIN-weights -for pallet in "${pallets[@]}"; do - output_file=./$CHAIN-weights/ - # a little hack for pallet_xcm_benchmarks - we want to output them to a nested directory - if [[ "$pallet" == "pallet_xcm_benchmarks::generic" ]] || [[ "$pallet" == "pallet_xcm_benchmarks::fungible" ]]; then - mkdir -p ./$CHAIN-weights/xcm - output_file="${output_file}xcm/${pallet//::/_}.rs" - fi - echo "Running benchmarks for $pallet to $output_file" - ./target/production/polkadot benchmark pallet \ - --chain=./$CHAIN-chain-spec.json \ - --steps=50 \ - --repeat=20 \ - --pallet=$pallet \ - --extrinsic=* \ - --wasm-execution=compiled \ - --heap-pages=4096 \ - --output="$output_file" \ - --header=./file_header.txt -done +/cmd --help # to see all available commands ``` -You probably want to do this inside a `tmux` session or something similar (e.g., `nohup &`), as it will take a while (several hours). - -7. `rsync` the weights back to your local machine, replacing the existing weights. - -8. Manually fix XCM weights by -- Replacing `impl xxx::yyy::WeightInfo for WeightInfo {` with `impl WeightInfo {` -- Marking all functions `pub(crate)` -- Removing any unused functions - -9. Commit the weight changes. +To generate weights for all pallets in a particular runtime(s), run the following command: +```sh +/cmd bench --runtime kusama polkadot +``` -10. Ensure the changes are reasonable. If not installed, `cargo install subweight`, check the weight changes: - ``` - subweight compare commits \ - --path-pattern "./**/weights/**/*.rs" \ - --method asymptotic \ - --ignore-errors \ - \ - ` - ``` - _Hint1: Add `--format markdown --no-color` for markdown-compatible results._ +> **πŸ“ Note**: The action is not being run right-away, it will be queued and run in the next available runner. So might be quick, but might also take up to 10 mins (That's in control of Github). +Once the action is run, you'll see reaction πŸ‘€ on original comment, and if you didn't pass `--quiet` - it will also send a link to a pipeline when started, and link to whole workflow when finished. - _Hint2: Change `--path-pattern "./**/weights/**/*.rs"` to e.g. `--path-pattern "./relay/polkadot/weights/**/*.rs"` for a specific runtime._ +> **πŸ’‘Hint #1** : if you run all runtimes or all pallets, it might be that some pallet in the middle is failed to generate weights, thus it stops (fails) the whole pipeline. +> If you want, you can make it to continue running, even if some pallets are failed, add `--continue-on-fail` flag to the command. The report will include which runtimes/pallets have failed, then you can re-run them separately after all is done. - _Hint3: Add `--change added changed` to include only relevant changes._ +This way it runs all possible runtimes for the specified pallets, if it finds them in the runtime +```sh +/cmd bench --pallet pallet_balances pallet_xcm_benchmarks::generic pallet_xcm_benchmarks::fungible +``` -## FAQ +If you want to run all specific pallet(s) for specific runtime(s), you can do it like this: +```sh +/cmd bench --runtime bridge-hub-polkadot --pallet pallet_xcm_benchmarks::generic pallet_xcm_benchmarks::fungible +``` -### What benchmark machine spec should I use? -See the [Polkadot Wiki Reference Hardware](https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#standard-hardware). +> **πŸ’‘Hint #2** : Sometimes when you run too many commands, or they keep failing and you're rerunning them again, it's handy to add `--clean` flag to the command. This will clean up all yours and bot's comments in PR relevant to /cmd commands. +```sh +/cmd bench --runtime kusama polkadot --pallet=pallet_balances --clean --continue-on-fail +``` diff --git a/relay/common/Cargo.toml b/relay/common/Cargo.toml index 1b6ded352a..0b735c6645 100644 --- a/relay/common/Cargo.toml +++ b/relay/common/Cargo.toml @@ -11,10 +11,10 @@ version.workspace = true codec = { features = ["derive", "max-encoded-len"], workspace = true } scale-info = { features = ["derive"], workspace = true } -sp-api ={ workspace = true } +sp-api = { workspace = true } sp-runtime = { workspace = true } polkadot-primitives = { workspace = true } -pallet-staking-reward-fn ={ workspace = true } +pallet-staking-reward-fn = { workspace = true } [features] @@ -23,8 +23,8 @@ std = [ "codec/std", "scale-info/std", + "pallet-staking-reward-fn/std", + "polkadot-primitives/std", "sp-api/std", "sp-runtime/std", - "polkadot-primitives/std", - "pallet-staking-reward-fn/std", ] diff --git a/relay/kusama/Cargo.toml b/relay/kusama/Cargo.toml index 0cc6705aab..bb35b40543 100644 --- a/relay/kusama/Cargo.toml +++ b/relay/kusama/Cargo.toml @@ -167,9 +167,9 @@ std = [ "pallet-nomination-pools/std", "pallet-offences-benchmarking?/std", "pallet-offences/std", + "pallet-parameters/std", "pallet-preimage/std", "pallet-proxy/std", - "pallet-parameters/std", "pallet-ranked-collective/std", "pallet-recovery/std", "pallet-referenda/std", @@ -301,6 +301,7 @@ try-runtime = [ "pallet-nis/try-runtime", "pallet-nomination-pools/try-runtime", "pallet-offences/try-runtime", + "pallet-parameters/try-runtime", "pallet-preimage/try-runtime", "pallet-proxy/try-runtime", "pallet-ranked-collective/try-runtime", @@ -320,7 +321,6 @@ try-runtime = [ "polkadot-runtime-common/try-runtime", "runtime-parachains/try-runtime", "sp-runtime/try-runtime", - "pallet-parameters/try-runtime", ] # Enable metadata hash generation at compile time for the `CheckMetadataHash` extension. diff --git a/relay/polkadot/Cargo.toml b/relay/polkadot/Cargo.toml index 6a936c2502..159497e65e 100644 --- a/relay/polkadot/Cargo.toml +++ b/relay/polkadot/Cargo.toml @@ -147,6 +147,7 @@ std = [ "pallet-beefy-mmr/std", "pallet-beefy/std", "pallet-bounties/std", + "pallet-broker/std", "pallet-child-bounties/std", "pallet-conviction-voting/std", "pallet-election-provider-multi-phase/std", @@ -182,6 +183,7 @@ std = [ "pallet-whitelist/std", "pallet-xcm-benchmarks?/std", "pallet-xcm/std", + "polkadot-parachain-primitives/std", "polkadot-primitives/std", "polkadot-runtime-common/std", "polkadot-runtime-constants/std", @@ -212,8 +214,6 @@ std = [ "xcm-executor/std", "xcm-runtime-apis/std", "xcm/std", - "pallet-broker/std", - "polkadot-parachain-primitives/std" ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", @@ -226,6 +226,7 @@ runtime-benchmarks = [ "pallet-bags-list/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-bounties/runtime-benchmarks", + "pallet-broker/runtime-benchmarks", "pallet-child-bounties/runtime-benchmarks", "pallet-conviction-voting/runtime-benchmarks", "pallet-election-provider-multi-phase/runtime-benchmarks", @@ -255,6 +256,7 @@ runtime-benchmarks = [ "pallet-whitelist/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "runtime-parachains/runtime-benchmarks", @@ -263,8 +265,6 @@ runtime-benchmarks = [ "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", "xcm-runtime-apis/runtime-benchmarks", - "pallet-broker/runtime-benchmarks", - "polkadot-parachain-primitives/runtime-benchmarks" ] try-runtime = [ "frame-election-provider-support/try-runtime", diff --git a/system-parachains/coretime/coretime-polkadot/Cargo.toml b/system-parachains/coretime/coretime-polkadot/Cargo.toml index e1a786d326..164d621dfe 100644 --- a/system-parachains/coretime/coretime-polkadot/Cargo.toml +++ b/system-parachains/coretime/coretime-polkadot/Cargo.toml @@ -105,7 +105,6 @@ std = [ "frame-system-rpc-runtime-api/std", "frame-system/std", "frame-try-runtime?/std", - "polkadot-runtime-constants/std", "log/std", "pallet-aura/std", "pallet-authorship/std", @@ -127,6 +126,7 @@ std = [ "polkadot-core-primitives/std", "polkadot-parachain-primitives/std", "polkadot-runtime-common/std", + "polkadot-runtime-constants/std", "scale-info/std", "serde", "serde_json/std", diff --git a/system-parachains/encointer/Cargo.toml b/system-parachains/encointer/Cargo.toml index 0ea8ab3b3d..4deb4218eb 100644 --- a/system-parachains/encointer/Cargo.toml +++ b/system-parachains/encointer/Cargo.toml @@ -141,8 +141,8 @@ runtime-benchmarks = [ "pallet-scheduler/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-utility/runtime-benchmarks", - "pallet-xcm/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", + "pallet-xcm/runtime-benchmarks", "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", @@ -201,8 +201,8 @@ std = [ "pallet-transaction-payment-rpc-runtime-api/std", "pallet-transaction-payment/std", "pallet-utility/std", - "pallet-xcm/std", "pallet-xcm-benchmarks?/std", + "pallet-xcm/std", "parachain-info/std", "parachains-common/std", "polkadot-core-primitives/std",