diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..99262ca894 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.dockerignore +.git/ +.github/ +.gitignore +.idea/ +README.md +Dockerfile +f +target/ diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index e64852e428..a9ba8d26a9 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -35,6 +35,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - uses: mozilla-actions/sccache-action@v0.0.4 with: diff --git a/.github/workflows/changelog-label.yml b/.github/workflows/changelog-label.yml index ffd8ec2103..3da79c6e91 100644 --- a/.github/workflows/changelog-label.yml +++ b/.github/workflows/changelog-label.yml @@ -13,6 +13,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: 'recursive' - name: Check if changes to CHANGELOG.md shell: bash env: diff --git a/.github/workflows/client-targets.yml b/.github/workflows/client-targets.yml index 1a33d2ae59..4d32555954 100644 --- a/.github/workflows/client-targets.yml +++ b/.github/workflows/client-targets.yml @@ -32,6 +32,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - run: cargo install cargo-ndk@2.12.2 @@ -56,6 +58,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - name: Setup Rust run: | diff --git a/.github/workflows/crate-check.yml b/.github/workflows/crate-check.yml index 6f130853ac..b92b182baa 100644 --- a/.github/workflows/crate-check.yml +++ b/.github/workflows/crate-check.yml @@ -19,6 +19,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: 'recursive' - name: Get commit range (push) if: ${{ github.event_name == 'push' }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c348d69acb..bb402372c6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,6 +22,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: 'recursive' - name: Get commit range (push) if: ${{ github.event_name == 'push' }} @@ -77,6 +78,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: 'recursive' - name: Setup Node uses: actions/setup-node@v4 diff --git a/.github/workflows/downstream-project-anchor.yml b/.github/workflows/downstream-project-anchor.yml index 33ecc632f0..6356428200 100644 --- a/.github/workflows/downstream-project-anchor.yml +++ b/.github/workflows/downstream-project-anchor.yml @@ -41,9 +41,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: ["v0.29.0", "v0.30.0"] + version: [ "v0.29.0", "v0.30.0" ] steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - shell: bash run: | diff --git a/.github/workflows/downstream-project-spl.yml b/.github/workflows/downstream-project-spl.yml index d2065f178f..421b2a49bd 100644 --- a/.github/workflows/downstream-project-spl.yml +++ b/.github/workflows/downstream-project-spl.yml @@ -40,6 +40,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - shell: bash run: | @@ -66,7 +68,7 @@ jobs: arrays: [ { - test_paths: ["token/cli"], + test_paths: [ "token/cli" ], required_programs: [ "token/program", @@ -76,14 +78,14 @@ jobs: ], }, { - test_paths: ["single-pool/cli"], + test_paths: [ "single-pool/cli" ], required_programs: [ "single-pool/program", ], }, { - test_paths: ["token-upgrade/cli"], + test_paths: [ "token-upgrade/cli" ], required_programs: [ "token-upgrade/program", @@ -92,6 +94,8 @@ jobs: ] steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - shell: bash run: | @@ -126,26 +130,28 @@ jobs: strategy: matrix: programs: - - [token/program] + - [ token/program ] - [ - instruction-padding/program, - token/program-2022, - token/program-2022-test, - ] + instruction-padding/program, + token/program-2022, + token/program-2022-test, + ] - [ - associated-token-account/program, - associated-token-account/program-test, - ] - - [token-upgrade/program] - - [feature-proposal/program] - - [governance/addin-mock/program, governance/program] - - [memo/program] - - [name-service/program] - - [stake-pool/program] - - [single-pool/program] + associated-token-account/program, + associated-token-account/program-test, + ] + - [ token-upgrade/program ] + - [ feature-proposal/program ] + - [ governance/addin-mock/program, governance/program ] + - [ memo/program ] + - [ name-service/program ] + - [ stake-pool/program ] + - [ single-pool/program ] steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' - shell: bash run: | diff --git a/.github/workflows/rebase.yaml b/.github/workflows/rebase.yaml new file mode 100644 index 0000000000..5fcc61fff4 --- /dev/null +++ b/.github/workflows/rebase.yaml @@ -0,0 +1,181 @@ +# This workflow runs a periodic rebase process, pulling in updates from an upstream repository +# The workflow for rebasing a jito-solana branch to a solana labs branch locally is typically: +# $ git checkout v1.17 +# $ git pull --rebase # --rebase needed locally +# $ git branch -D lb/v1.17_rebase # deletes branch from last v1.17 rebase +# $ git checkout -b lb/v1.17_rebase +# $ git fetch upstream +# $ git rebase upstream/v1.17 # rebase + fix merge conflicts +# $ git rebase --continue +# $ git push origin +lb/v1.17_rebase # force needed to overwrite remote. wait for CI, fix if any issues +# $ git checkout v1.17 +# $ git reset --hard lb/v1.17_rebase +# $ git push origin +v1.17 +# +# This workflow automates this process, with periodic status updates over slack. +# It will also run CI and wait for it to pass before performing the force push to v1.17. +# In the event there's a failure in the process, it's reported to slack and the job stops. + +name: "Rebase jito-solana from upstream anza-xyz/agave" + +on: + # push: + schedule: + - cron: "30 18 * * 1-5" + +jobs: + rebase: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - branch: master + rebase: upstream/master + - branch: v1.18 + rebase: upstream/v1.18 + - branch: v1.17 + rebase: upstream/v1.17 + # note: this will always be a day behind because we're rebasing from the previous day's rebase + # and NOT upstream + - branch: v1.17-fast-replay + rebase: origin/v1.17 + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ matrix.branch }} + submodules: recursive + fetch-depth: 0 + token: ${{ secrets.JITO_SOLANA_RELEASE_TOKEN }} + - name: Add upstream + run: git remote add upstream https://github.com/anza-xyz/agave.git + - name: Fetch upstream + run: git fetch upstream + - name: Fetch origin + run: git fetch origin + - name: Set REBASE_BRANCH + run: echo "REBASE_BRANCH=ci/nightly/${{ matrix.branch }}/$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_ENV + - name: echo $REBASE_BRANCH + run: echo $REBASE_BRANCH + - name: Create rebase branch + run: git checkout -b $REBASE_BRANCH + - name: Setup email + run: | + git config --global user.email "infra@jito.wtf" + git config --global user.name "Jito Infrastructure" + - name: Rebase + id: rebase + run: git rebase ${{ matrix.rebase }} + - name: Send warning for rebase error + if: failure() && steps.rebase.outcome == 'failure' + uses: slackapi/slack-github-action@v1.25.0 + with: + payload: | + { + "text": "Nightly rebase on branch ${{ matrix.branch }}\nStatus: Rebase failed to apply cleanly" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + - name: Check if rebase applied + id: check_rebase_applied + run: | + PRE_REBASE_SHA=$(git rev-parse ${{ matrix.branch }}) + POST_REBASE_SHA=$(git rev-parse HEAD) + if [ "$PRE_REBASE_SHA" = "$POST_REBASE_SHA" ]; then + echo "No rebase was applied, exiting..." + exit 1 + else + echo "Rebase applied successfully." + fi + - name: Send warning for rebase error + if: failure() && steps.check_rebase_applied.outcome == 'failure' + uses: slackapi/slack-github-action@v1.25.0 + with: + payload: | + { + "text": "Nightly rebase on branch ${{ matrix.branch }}\nStatus: Rebase not needed" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + - name: Set REBASE_SHA + run: echo "REBASE_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: ${{ env.REBASE_BRANCH }} + - name: Wait for buildkite to start build + run: sleep 300 + - name: Wait for buildkite to finish + id: wait_for_buildkite + timeout-minutes: 300 + run: | + while true; do + response=$(curl -s -f -H "Authorization: Bearer ${{ secrets.BUILDKITE_TOKEN }}" "https://api.buildkite.com/v2/organizations/jito/pipelines/jito-solana/builds?commit=${{ env.REBASE_SHA }}") + if [ $? -ne 0 ]; then + echo "Curl request failed." + exit 1 + fi + + state=$(echo $response | jq --exit-status -r '.[0].state') + echo "Current build state: $state" + + # Check if the state is one of the finished states + case $state in + "passed"|"finished") + echo "Build finished successfully." + exit 0 + ;; + "canceled"|"canceling"|"not_run") + # ignoring "failing"|"failed" because flaky CI, can restart and hope it finishes or times out + echo "Build failed or was cancelled." + exit 2 + ;; + esac + + sleep 30 + done + - name: Send failure update + uses: slackapi/slack-github-action@v1.25.0 + if: failure() && steps.wait_for_buildkite.outcome == 'failure' + with: + payload: | + { + "text": "Nightly rebase on branch ${{ matrix.branch }}\nStatus: CI failed\nBranch: ${{ env.REBASE_BRANCH}}\nBuild: https://buildkite.com/jito/jito-solana/builds?commit=${{ env.REBASE_SHA }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # check to see if different branch since CI build can take awhile and these steps are not atomic + - name: Fetch the latest remote changes + run: git fetch origin ${{ matrix.branch }} + - name: Check if origin HEAD has changed from the beginning of the workflow + run: | + LOCAL_SHA=$(git rev-parse ${{ matrix.branch }}) + ORIGIN_SHA=$(git rev-parse origin/${{ matrix.branch }}) + if [ "$ORIGIN_SHA" != "$LOCAL_SHA" ]; then + echo "The remote HEAD of ${{ matrix.branch }} does not match the local HEAD of ${{ matrix.branch }} at the beginning of CI." + echo "origin sha: $ORIGIN_SHA" + echo "local sha: $LOCAL_SHA" + exit 1 + else + echo "The remote HEAD matches the local REBASE_SHA at the beginning of CI. Proceeding." + fi + - name: Reset ${{ matrix.branch }} to ${{ env.REBASE_BRANCH }} + run: | + git checkout ${{ matrix.branch }} + git reset --hard ${{ env.REBASE_BRANCH }} + - name: Push rebased %{{ matrix.branch }} + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.JITO_SOLANA_RELEASE_TOKEN }} + branch: ${{ matrix.branch }} + force: true + - name: Send success update + uses: slackapi/slack-github-action@v1.25.0 + with: + payload: | + { + "text": "Nightly rebase on branch ${{ matrix.branch }}\nStatus: CI success, rebased, and pushed\nBranch: ${{ env.REBASE_BRANCH}}\nBuild: https://buildkite.com/jito/jito-solana/builds?commit=${{ env.REBASE_SHA }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index de32cee71d..0543c964de 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -22,6 +22,7 @@ jobs: with: ref: master fetch-depth: 0 + submodules: 'recursive' - name: Setup Rust shell: bash diff --git a/.gitignore b/.gitignore index 3127645a25..c775b49cc5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ target/ /solana-release.tar.bz2 /solana-metrics/ /solana-metrics.tar.bz2 +**/target/ /test-ledger/ **/*.rs.bk @@ -28,7 +29,11 @@ log-*/ # fetch-spl.sh artifacts /spl-genesis-args.sh /spl_*.so +/jito_*.so .DS_Store # scripts that may be generated by cargo *-bpf commands **/cargo-*-bpf-child-script-*.sh + +.env +docker-output/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..e31fc7fccd --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "anchor"] + path = anchor + url = https://github.com/jito-foundation/anchor.git +[submodule "jito-programs"] + path = jito-programs + url = https://github.com/jito-foundation/jito-programs.git +[submodule "jito-protos/protos"] + path = jito-protos/protos + url = https://github.com/jito-labs/mev-protos.git diff --git a/Cargo.lock b/Cargo.lock index 33082d2170..dcdb6dc869 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,7 +156,7 @@ name = "agave-ledger-tool" version = "2.1.0" dependencies = [ "assert_cmd", - "bs58", + "bs58 0.5.1", "bytecount", "chrono", "clap 2.33.3", @@ -271,6 +271,7 @@ dependencies = [ "solana-rpc-client", "solana-rpc-client-api", "solana-runtime", + "solana-runtime-plugin", "solana-sdk", "solana-send-transaction-service", "solana-storage-bigtable", @@ -286,6 +287,7 @@ dependencies = [ "thiserror", "tikv-jemallocator", "tokio", + "tonic", ] [[package]] @@ -370,6 +372,145 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "anchor-attribute-access-control" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "regex", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-account" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "bs58 0.4.0", + "proc-macro2 1.0.86", + "quote 1.0.36", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-constant" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "proc-macro2 1.0.86", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-error" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-event" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-interface" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "heck 0.3.3", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-program" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-state" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-derive-accounts" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-lang" +version = "0.24.2" +dependencies = [ + "anchor-attribute-access-control", + "anchor-attribute-account", + "anchor-attribute-constant", + "anchor-attribute-error", + "anchor-attribute-event", + "anchor-attribute-interface", + "anchor-attribute-program", + "anchor-attribute-state", + "anchor-derive-accounts", + "arrayref", + "base64 0.13.1", + "bincode", + "borsh 0.10.3", + "bytemuck", + "solana-program", + "thiserror", +] + +[[package]] +name = "anchor-syn" +version = "0.24.2" +dependencies = [ + "anyhow", + "bs58 0.3.1", + "heck 0.3.3", + "proc-macro2 1.0.86", + "proc-macro2-diagnostics", + "quote 1.0.36", + "serde", + "serde_json", + "sha2 0.9.9", + "syn 1.0.109", + "thiserror", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -400,12 +541,55 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + [[package]] name = "anyhow" version = "1.0.86" @@ -421,8 +605,8 @@ dependencies = [ "include_dir", "itertools 0.10.5", "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -495,7 +679,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" dependencies = [ - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -507,8 +691,8 @@ checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ "num-bigint 0.4.6", "num-traits", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -543,8 +727,8 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae3281bc6d0fd7e549af32b52511e1302185bd688fd3359fa36423346ff682ea" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -604,8 +788,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "synstructure", ] @@ -616,8 +800,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -697,8 +881,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -708,8 +892,8 @@ version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -858,8 +1042,8 @@ dependencies = [ "itertools 0.12.1", "lazy_static", "lazycell", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "regex", "rustc-hash", "shlex", @@ -1006,7 +1190,7 @@ dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", - "proc-macro2", + "proc-macro2 1.0.86", "syn 1.0.109", ] @@ -1018,8 +1202,8 @@ checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" dependencies = [ "once_cell", "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", "syn_derive", ] @@ -1030,8 +1214,8 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1041,8 +1225,8 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1067,6 +1251,18 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb" + +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + [[package]] name = "bs58" version = "0.5.1" @@ -1150,8 +1346,8 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -1389,7 +1585,7 @@ checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", "bitflags 1.3.2", - "clap_derive", + "clap_derive 3.2.18", "clap_lex 0.2.4", "indexmap 1.9.3", "once_cell", @@ -1405,6 +1601,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" dependencies = [ "clap_builder", + "clap_derive 4.3.12", + "once_cell", ] [[package]] @@ -1413,8 +1611,10 @@ version = "4.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" dependencies = [ + "anstream", "anstyle", "clap_lex 0.5.0", + "strsim 0.10.0", ] [[package]] @@ -1423,13 +1623,25 @@ version = "3.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" dependencies = [ - "heck", + "heck 0.4.0", "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] +[[package]] +name = "clap_derive" +version = "4.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +dependencies = [ + "heck 0.4.0", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 2.0.71", +] + [[package]] name = "clap_lex" version = "0.2.4" @@ -1445,6 +1657,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + [[package]] name = "combine" version = "3.8.1" @@ -1515,9 +1733,9 @@ version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.86", + "quote 1.0.36", + "unicode-xid 0.2.2", ] [[package]] @@ -1766,8 +1984,8 @@ checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "strsim 0.10.0", "syn 2.0.71", ] @@ -1779,7 +1997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core", - "quote", + "quote 1.0.36", "syn 2.0.71", ] @@ -1803,6 +2021,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +[[package]] +name = "default-env" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f753eb82d29277e79efc625e84aecacfd4851ee50e05a8573a4740239a77bfd3" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "der-parser" version = "8.1.0" @@ -1829,8 +2058,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1840,8 +2069,8 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -1852,8 +2081,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df" dependencies = [ "convert_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustc_version 0.3.3", "syn 1.0.109", ] @@ -1941,8 +2170,8 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1964,8 +2193,8 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cbae11b3de8fce2a456e8ea3dada226b35fe791f0dc1d360c0941f0bb681f3" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -2029,8 +2258,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86b50932a01e7ec5c06160492ab660fb19b6bb2a7878030dd6cd68d21df9d4d" dependencies = [ "enum-ordinalize", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2070,8 +2299,8 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -2083,8 +2312,8 @@ checksum = "0b166c9e378360dd5a6666a9604bb4f54ae0cac39023ffbac425e917a2a04fef" dependencies = [ "num-bigint 0.4.6", "num-traits", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2340,8 +2569,8 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -2616,6 +2845,15 @@ dependencies = [ "http", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.0" @@ -2890,8 +3128,8 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b139284b5cf57ecfa712bcc66950bb635b31aff41c188e8a4cfc758eca374a3f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", ] [[package]] @@ -2993,6 +3231,49 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jito-programs-vote-state" +version = "0.1.5" +dependencies = [ + "anchor-lang", + "bincode", + "serde", + "serde_derive", + "solana-program", +] + +[[package]] +name = "jito-protos" +version = "2.1.0" +dependencies = [ + "bytes", + "prost", + "prost-types", + "protobuf-src", + "tonic", + "tonic-build", +] + +[[package]] +name = "jito-tip-distribution" +version = "0.1.5" +dependencies = [ + "anchor-lang", + "default-env", + "jito-programs-vote-state", + "solana-program", + "solana-security-txt", +] + +[[package]] +name = "jito-tip-payment" +version = "0.1.5" +dependencies = [ + "anchor-lang", + "default-env", + "solana-security-txt", +] + [[package]] name = "jobserver" version = "0.1.24" @@ -3073,8 +3354,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" dependencies = [ "proc-macro-crate 0.1.5", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3486,8 +3767,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ "cfg-if 1.0.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3507,8 +3788,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3628,8 +3909,8 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -3701,8 +3982,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -3784,8 +4065,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3991,8 +4272,8 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest", "pest_meta", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -4042,8 +4323,8 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -4159,7 +4440,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b83ec2d0af5c5c556257ff52c9f98934e243b9fd39604bfb2a9b75ec2e97f18" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.86", "syn 1.0.109", ] @@ -4194,8 +4475,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "version_check", ] @@ -4206,11 +4487,20 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "version_check", ] +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -4220,6 +4510,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada" +dependencies = [ + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", + "version_check", + "yansi", +] + [[package]] name = "proptest" version = "1.5.0" @@ -4257,7 +4560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", - "heck", + "heck 0.4.0", "itertools 0.10.5", "lazy_static", "log", @@ -4280,8 +4583,8 @@ checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -4326,8 +4629,8 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -4385,13 +4688,22 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.86", ] [[package]] @@ -4828,7 +5140,7 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.3", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] @@ -4862,6 +5174,16 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "rustls-webpki" +version = "0.100.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" +dependencies = [ + "ring 0.16.20", + "untrusted 0.7.1", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -5026,8 +5348,8 @@ version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -5080,8 +5402,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" dependencies = [ "darling", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -5130,8 +5452,8 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -5386,7 +5708,7 @@ dependencies = [ "assert_matches", "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "bv", "lazy_static", "serde", @@ -5604,6 +5926,7 @@ dependencies = [ "futures 0.3.30", "solana-banks-interface", "solana-client", + "solana-gossip", "solana-runtime", "solana-sdk", "solana-send-transaction-service", @@ -5764,6 +6087,29 @@ dependencies = [ "tempfile", ] +[[package]] +name = "solana-bundle" +version = "2.1.0" +dependencies = [ + "anchor-lang", + "assert_matches", + "itertools 0.12.1", + "log", + "serde", + "solana-accounts-db", + "solana-ledger", + "solana-logger", + "solana-measure", + "solana-poh", + "solana-program-runtime", + "solana-runtime", + "solana-sdk", + "solana-svm", + "solana-timings", + "solana-transaction-status", + "thiserror", +] + [[package]] name = "solana-cargo-build-sbf" version = "2.1.0" @@ -5838,7 +6184,7 @@ version = "2.1.0" dependencies = [ "assert_matches", "bincode", - "bs58", + "bs58 0.5.1", "clap 2.33.3", "console", "const_format", @@ -6056,11 +6402,12 @@ name = "solana-core" version = "2.1.0" dependencies = [ "ahash 0.8.10", + "anchor-lang", "arrayvec", "assert_matches", "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "bytes", "chrono", "crossbeam-channel", @@ -6070,12 +6417,17 @@ dependencies = [ "futures 0.3.30", "histogram", "itertools 0.12.1", + "jito-protos", + "jito-tip-distribution", + "jito-tip-payment", "lazy_static", "log", "lru", "min-max-heap", "num_enum", "prio-graph", + "prost", + "prost-types", "qualifier_attr", "quinn", "rand 0.8.5", @@ -6091,6 +6443,7 @@ dependencies = [ "serial_test", "solana-accounts-db", "solana-bloom", + "solana-bundle", "solana-client", "solana-compute-budget", "solana-connection-cache", @@ -6109,11 +6462,13 @@ dependencies = [ "solana-perf", "solana-poh", "solana-program-runtime", + "solana-program-test", "solana-quic-client", "solana-rayon-threadlimit", "solana-rpc", "solana-rpc-client-api", "solana-runtime", + "solana-runtime-plugin", "solana-sanitize", "solana-sdk", "solana-send-transaction-service", @@ -6140,6 +6495,8 @@ dependencies = [ "test-case", "thiserror", "tokio", + "tonic", + "tonic-build", "trees", ] @@ -6298,7 +6655,7 @@ name = "solana-frozen-abi" version = "2.1.0" dependencies = [ "bitflags 2.6.0", - "bs58", + "bs58 0.5.1", "bv", "generic-array 0.14.7", "im", @@ -6318,8 +6675,8 @@ dependencies = [ name = "solana-frozen-abi-macro" version = "2.1.0" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustc_version 0.4.0", "syn 2.0.71", ] @@ -6365,7 +6722,7 @@ name = "solana-geyser-plugin-manager" version = "2.1.0" dependencies = [ "agave-geyser-plugin-interface", - "bs58", + "bs58 0.5.1", "crossbeam-channel", "json5", "jsonrpc-core", @@ -6450,7 +6807,7 @@ dependencies = [ name = "solana-keygen" version = "2.1.0" dependencies = [ - "bs58", + "bs58 0.5.1", "clap 3.2.23", "dirs-next", "num_cpus", @@ -6482,7 +6839,7 @@ dependencies = [ "assert_matches", "bincode", "bitflags 2.6.0", - "bs58", + "bs58 0.5.1", "byteorder", "chrono", "chrono-humanize", @@ -6735,8 +7092,8 @@ dependencies = [ name = "solana-package-metadata-macro" version = "2.1.0" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", "toml 0.8.12", ] @@ -6835,7 +7192,7 @@ dependencies = [ "blake3", "borsh 0.10.3", "borsh 1.5.1", - "bs58", + "bs58 0.5.1", "bv", "bytemuck", "bytemuck_derive", @@ -7029,7 +7386,7 @@ version = "2.1.0" dependencies = [ "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "crossbeam-channel", "dashmap", "itertools 0.12.1", @@ -7049,6 +7406,7 @@ dependencies = [ "soketto", "solana-account-decoder", "solana-accounts-db", + "solana-bundle", "solana-client", "solana-entry", "solana-faucet", @@ -7060,6 +7418,7 @@ dependencies = [ "solana-net-utils", "solana-perf", "solana-poh", + "solana-program-runtime", "solana-rayon-threadlimit", "solana-rpc-client-api", "solana-runtime", @@ -7092,7 +7451,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "crossbeam-channel", "futures 0.3.30", "indicatif", @@ -7120,7 +7479,7 @@ version = "2.1.0" dependencies = [ "anyhow", "base64 0.22.1", - "bs58", + "bs58 0.5.1", "const_format", "jsonrpc-core", "reqwest", @@ -7130,8 +7489,10 @@ dependencies = [ "serde_derive", "serde_json", "solana-account-decoder", + "solana-bundle", "solana-inline-spl", "solana-sdk", + "solana-svm", "solana-transaction-status", "solana-version", "thiserror", @@ -7159,13 +7520,14 @@ name = "solana-rpc-test" version = "2.1.0" dependencies = [ "bincode", - "bs58", + "bs58 0.5.1", "crossbeam-channel", "futures-util", "log", "reqwest", "serde", "serde_json", + "serial_test", "solana-account-decoder", "solana-client", "solana-connection-cache", @@ -7271,6 +7633,24 @@ dependencies = [ "zstd", ] +[[package]] +name = "solana-runtime-plugin" +version = "2.1.0" +dependencies = [ + "crossbeam-channel", + "json5", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "libloading", + "log", + "solana-runtime", + "solana-sdk", + "thiserror", +] + [[package]] name = "solana-runtime-transaction" version = "2.1.0" @@ -7296,12 +7676,14 @@ dependencies = [ name = "solana-sdk" version = "2.1.0" dependencies = [ + "anchor-lang", "anyhow", "assert_matches", + "base64 0.22.1", "bincode", "bitflags 2.6.0", "borsh 1.5.1", - "bs58", + "bs58 0.5.1", "bytemuck", "bytemuck_derive", "byteorder", @@ -7358,9 +7740,9 @@ dependencies = [ name = "solana-sdk-macro" version = "2.1.0" dependencies = [ - "bs58", - "proc-macro2", - "quote", + "bs58 0.5.1", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -7392,11 +7774,13 @@ dependencies = [ "log", "solana-client", "solana-connection-cache", + "solana-gossip", "solana-logger", "solana-measure", "solana-metrics", "solana-runtime", "solana-sdk", + "solana-streamer", "solana-tpu-client", ] @@ -7499,7 +7883,7 @@ name = "solana-storage-proto" version = "2.1.0" dependencies = [ "bincode", - "bs58", + "bs58 0.5.1", "enum-iterator", "prost", "protobuf-src", @@ -7667,6 +8051,44 @@ dependencies = [ "solana-sdk", ] +[[package]] +name = "solana-tip-distributor" +version = "2.1.0" +dependencies = [ + "anchor-lang", + "clap 4.3.21", + "crossbeam-channel", + "env_logger", + "futures 0.3.30", + "gethostname", + "im", + "itertools 0.12.1", + "jito-tip-distribution", + "jito-tip-payment", + "log", + "num-traits", + "rand 0.8.5", + "serde", + "serde_json", + "solana-accounts-db", + "solana-client", + "solana-genesis-utils", + "solana-ledger", + "solana-measure", + "solana-merkle-tree", + "solana-metrics", + "solana-program", + "solana-program-runtime", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-stake-program", + "solana-transaction-status", + "solana-vote", + "thiserror", + "tokio", +] + [[package]] name = "solana-tokens" version = "2.1.0" @@ -7793,7 +8215,7 @@ dependencies = [ "base64 0.22.1", "bincode", "borsh 1.5.1", - "bs58", + "bs58 0.5.1", "lazy_static", "log", "serde", @@ -8010,7 +8432,7 @@ dependencies = [ name = "solana-zk-keygen" version = "2.1.0" dependencies = [ - "bs58", + "bs58 0.5.1", "clap 3.2.23", "dirs-next", "solana-clap-v3-utils", @@ -8172,7 +8594,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9e8418ea6269dcfb01c712f0444d2c75542c04448b480e87de59d2865edc750" dependencies = [ - "quote", + "quote 1.0.36", "spl-discriminator-syn", "syn 2.0.71", ] @@ -8183,8 +8605,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c1f05593b7ca9eac7caca309720f2eafb96355e037e6d373b909a80fe7b69b9" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "sha2 0.10.8", "syn 2.0.71", "thiserror", @@ -8242,8 +8664,8 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d375dd76c517836353e093c2dbb490938ff72821ab568b545fd30ab3256b3e" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "sha2 0.10.8", "syn 2.0.71", ] @@ -8401,9 +8823,9 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck", - "proc-macro2", - "quote", + "heck 0.4.0", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustversion", "syn 1.0.109", ] @@ -8420,14 +8842,25 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "unicode-ident", ] @@ -8437,8 +8870,8 @@ version = "2.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "unicode-ident", ] @@ -8449,8 +8882,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" dependencies = [ "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -8466,10 +8899,10 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", - "unicode-xid", + "unicode-xid 0.2.2", ] [[package]] @@ -8577,8 +9010,8 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee42b4e559f17bce0385ebf511a7beb67d5cc33c12c96b7f4e9789919d9c10f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -8635,8 +9068,8 @@ checksum = "54c25e2cb8f5fcd7318157634e8838aa6f7e4715c96637f969fabaccd1ef5462" dependencies = [ "cfg-if 1.0.0", "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -8647,8 +9080,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37cfd7bbc88a0104e304229fba519bdc45501a30b760fb72240342f1289ad257" dependencies = [ "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", "test-case-core", ] @@ -8683,8 +9116,8 @@ version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -8820,8 +9253,8 @@ name = "tokio-macros" version = "2.1.0" source = "git+https://github.com/anza-xyz/solana-tokio.git?rev=7cf47705faacf7bf0e43e4131a5377b3291fce21#7cf47705faacf7bf0e43e4131a5377b3291fce21" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -8991,6 +9424,7 @@ dependencies = [ "percent-encoding 2.3.1", "pin-project", "prost", + "rustls-native-certs", "rustls-pemfile 1.0.0", "tokio", "tokio-rustls", @@ -8999,6 +9433,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "webpki-roots 0.23.1", ] [[package]] @@ -9008,9 +9443,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" dependencies = [ "prettyplease", - "proc-macro2", + "proc-macro2 1.0.86", "prost-build", - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -9064,8 +9499,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -9183,12 +9618,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + [[package]] name = "unicode-width" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.2" @@ -9276,6 +9723,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cf7d77f457ef8dfa11e4cd5933c5ddb5dc52a94664071951219a97710f0a32b" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.0" @@ -9373,8 +9826,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", "wasm-bindgen-shared", ] @@ -9397,7 +9850,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ - "quote", + "quote 1.0.36", "wasm-bindgen-macro-support", ] @@ -9407,8 +9860,8 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -9430,13 +9883,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" +dependencies = [ + "rustls-webpki 0.100.3", +] + [[package]] name = "webpki-roots" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki", + "rustls-webpki 0.101.7", ] [[package]] @@ -9742,6 +10204,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zerocopy" version = "0.7.31" @@ -9757,8 +10225,8 @@ version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] @@ -9777,8 +10245,8 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.71", ] diff --git a/Cargo.toml b/Cargo.toml index 1d7f2d3549..2355403e42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "bench-tps", "bloom", "bucket_map", + "bundle", "cargo-registry", "clap-utils", "clap-v3-utils", @@ -46,6 +47,7 @@ members = [ "gossip", "inline-spl", "install", + "jito-protos", "keygen", "lattice-hash", "ledger", @@ -95,6 +97,7 @@ members = [ "rpc-client-nonce-utils", "rpc-test", "runtime", + "runtime-plugin", "runtime-transaction", "sanitize", "sdk", @@ -119,6 +122,7 @@ members = [ "test-validator", "thin-client", "timings", + "tip-distributor", "tokens", "tps-client", "tpu-client", @@ -141,7 +145,12 @@ members = [ "zk-token-sdk", ] -exclude = ["programs/sbf", "svm/tests/example-programs"] +exclude = [ + "anchor", + "jito-programs", + "programs/sbf", + "svm/tests/example-programs", +] resolver = "2" @@ -158,6 +167,7 @@ Inflector = "0.11.4" aquamarine = "0.3.3" aes-gcm-siv = "0.11.1" ahash = "0.8.10" +anchor-lang = { path = "anchor/lang" } anyhow = "1.0.82" arbitrary = "1.3.2" ark-bn254 = "0.4.0" @@ -247,6 +257,9 @@ jemallocator = { package = "tikv-jemallocator", version = "0.4.1", features = [ "unprefixed_malloc_on_supported_platforms", ] } js-sys = "0.3.69" +jito-protos = { path = "jito-protos", version = "=2.1.0" } +jito-tip-distribution = { path = "jito-programs/mev-programs/programs/tip-distribution", features = ["no-entrypoint"] } +jito-tip-payment = { path = "jito-programs/mev-programs/programs/tip-payment", features = ["no-entrypoint"] } json5 = "0.4.1" jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" @@ -254,6 +267,7 @@ jsonrpc-derive = "18.0.0" jsonrpc-http-server = "18.0.0" jsonrpc-ipc-server = "18.0.0" jsonrpc-pubsub = "18.0.0" +jsonrpc-server-utils = "18.0.0" lazy-lru = "0.1.2" lazy_static = "1.5.0" libc = "0.2.155" @@ -340,6 +354,7 @@ solana-bloom = { path = "bloom", version = "=2.1.0" } solana-bn254 = { path = "curves/bn254", version = "=2.1.0" } solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=2.1.0" } solana-bucket-map = { path = "bucket_map", version = "=2.1.0" } +solana-bundle = { path = "bundle", version = "=2.1.0" } agave-cargo-registry = { path = "cargo-registry", version = "=2.1.0" } solana-clap-utils = { path = "clap-utils", version = "=2.1.0" } solana-clap-v3-utils = { path = "clap-v3-utils", version = "=2.1.0" } @@ -399,6 +414,7 @@ solana-rpc-client = { path = "rpc-client", version = "=2.1.0", default-features solana-rpc-client-api = { path = "rpc-client-api", version = "=2.1.0" } solana-rpc-client-nonce-utils = { path = "rpc-client-nonce-utils", version = "=2.1.0" } solana-runtime = { path = "runtime", version = "=2.1.0" } +solana-runtime-plugin = { path = "runtime-plugin", version = "=2.1.0" } solana-runtime-transaction = { path = "runtime-transaction", version = "=2.1.0" } solana-sdk = { path = "sdk", version = "=2.1.0" } solana-sdk-macro = { path = "sdk/macro", version = "=2.1.0" } diff --git a/README.md b/README.md index 0d855e8cc8..4369c4bcb4 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,16 @@

-[![Solana crate](https://img.shields.io/crates/v/solana-core.svg)](https://crates.io/crates/solana-core) -[![Solana documentation](https://docs.rs/solana-core/badge.svg)](https://docs.rs/solana-core) -[![Build status](https://badge.buildkite.com/8cc350de251d61483db98bdfc895b9ea0ac8ffa4a32ee850ed.svg?branch=master)](https://buildkite.com/solana-labs/solana/builds?branch=master) -[![codecov](https://codecov.io/gh/solana-labs/solana/branch/master/graph/badge.svg)](https://codecov.io/gh/solana-labs/solana) +[![Build status](https://badge.buildkite.com/3a7c88c0f777e1a0fddacc190823565271ae4c251ef78d83a8.svg)](https://buildkite.com/jito/jito-solana) -# Building +# About + +This repository contains Jito's fork of the Solana validator. + +We recommend checking out our [Gitbook](https://jito-foundation.gitbook.io/mev/jito-solana/building-the-software) for +more detailed instructions on building and running Jito-Solana. + +--- ## **1. Install rustc, cargo and rustfmt.** @@ -25,21 +29,27 @@ When building the master branch, please make sure you are using the latest stabl $ rustup update ``` -When building a specific release branch, you should check the rust version in `ci/rust-version.sh` and if necessary, install that version by running: +When building a specific release branch, you should check the rust version in `ci/rust-version.sh` and if necessary, +install that version by running: + ```bash $ rustup install VERSION ``` -Note that if this is not the latest rust version on your machine, cargo commands may require an [override](https://rust-lang.github.io/rustup/overrides.html) in order to use the correct version. + +Note that if this is not the latest rust version on your machine, cargo commands may require +an [override](https://rust-lang.github.io/rustup/overrides.html) in order to use the correct version. On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, protobuf etc. On Ubuntu: + ```bash $ sudo apt-get update $ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang cmake make libprotobuf-dev protobuf-compiler ``` On Fedora: + ```bash $ sudo dnf install openssl-devel systemd-devel pkg-config zlib-devel llvm clang cmake make protobuf-devel protobuf-compiler perl-core ``` @@ -47,8 +57,8 @@ $ sudo dnf install openssl-devel systemd-devel pkg-config zlib-devel llvm clang ## **2. Download the source code.** ```bash -$ git clone https://github.com/anza-xyz/agave.git -$ cd agave +$ git clone https://github.com/jito-foundation/jito-solana.git +$ cd jito-solana ``` ## **3. Build.** @@ -72,7 +82,7 @@ Start your own testnet locally, instructions are in the [online docs](https://do ### Accessing the remote development cluster * `devnet` - stable public cluster for development accessible via -devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://docs.solanalabs.com/clusters) + devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://docs.solanalabs.com/clusters) # Benchmarking @@ -104,7 +114,7 @@ $ open target/cov/lcov-local/index.html Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to -some problem. Our unit-test suite is how we encode the set of *problems* the codebase solves. Running +some problem. Our unit-test suite is how we encode the set of *problems* the codebase solves. Running the test suite should indicate that your change didn't *infringe* on anyone else's solutions. Adding a test *protects* your solution from future changes. Say you don't understand why a line of code exists, try deleting it and running the unit-tests. The nearest test failure should tell you what problem @@ -113,3 +123,4 @@ problem is solved by this code?" On the other hand, if a test does fail and you better way to solve the same problem, a Pull Request with your solution would most certainly be welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please send us that patch! + diff --git a/RELEASE.md b/RELEASE.md index a862a0e410..9439ca3aa3 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,9 +17,10 @@ ``` ### master branch + All new development occurs on the `master` branch. -Bug fixes that affect a `vX.Y` branch are first made on `master`. This is to +Bug fixes that affect a `vX.Y` branch are first made on `master`. This is to allow a fix some soak time on `master` before it is applied to one or more stabilization branches. @@ -29,7 +30,7 @@ release blocker in a branch causes you to forget to propagate back to `master`!)" Once the bug fix lands on `master` it is cherry-picked into the `vX.Y` branch -and potentially the `vX.Y-1` branch. The exception to this rule is when a bug +and potentially the `vX.Y-1` branch. The exception to this rule is when a bug fix for `vX.Y` doesn't apply to `master` or `vX.Y-1`. Immediately after a new stabilization branch is forged, the `Cargo.toml` minor @@ -38,10 +39,12 @@ Incrementing the major version of the `master` branch is outside the scope of this document. ### v*X.Y* stabilization branches + These are stabilization branches. They are created from the `master` branch approximately every 13 weeks. ### v*X.Y.Z* release tag + The release tags are created as desired by the owner of the given stabilization branch, and cause that *X.Y.Z* release to be shipped to https://crates.io @@ -50,11 +53,13 @@ patch version number (*Z*) of the stabilization branch is incremented by the release engineer. ## Channels + Channels are used by end-users (humans and bots) to consume the branches described in the previous section, so they may automatically update to the most recent version matching their desired stability. There are three release channels that map to branches as follows: + * edge - tracks the `master` branch, least stable. * beta - tracks the largest (and latest) `vX.Y` stabilization branch, more stable. * stable - tracks the second largest `vX.Y` stabilization branch, most stable. @@ -62,18 +67,20 @@ There are three release channels that map to branches as follows: ## Steps to Create a Branch ### Major release branch + 1. If the new branch will be the first branch of a new major release check that -all eligible deprecated symbols have been removed. Our policy is to deprecate -for at least one full minor version before removal. + all eligible deprecated symbols have been removed. Our policy is to deprecate + for at least one full minor version before removal. ### Create the new branch + 1. Check out the latest commit on `master` branch: ``` git fetch --all git checkout upstream/master ``` -1. Determine the new branch name. The name should be "v" + the first 2 version fields - from Cargo.toml. For example, a Cargo.toml with version = "0.9.0" implies +1. Determine the new branch name. The name should be "v" + the first 2 version fields + from Cargo.toml. For example, a Cargo.toml with version = "0.9.0" implies the next branch name is "v0.9". 1. Create the new branch and push this branch to the `agave` repository: ``` @@ -85,7 +92,8 @@ Alternatively use the Github UI. ### Update master branch to the next release minor version -1. After the new branch has been created and pushed, update the Cargo.toml files on **master** to the next semantic version (e.g. 0.9.0 -> 0.10.0) with: +1. After the new branch has been created and pushed, update the Cargo.toml files on **master** to the next semantic + version (e.g. 0.9.0 -> 0.10.0) with: ``` $ scripts/increment-cargo-version.sh minor ``` @@ -96,60 +104,82 @@ Alternatively use the Github UI. git commit -m 'Bump version to X.Y+1.0' git push -u origin version_update ``` -1. Confirm that your freshly cut release branch is shown as `BETA_CHANNEL` and the previous release branch as `STABLE_CHANNEL`: +1. Confirm that your freshly cut release branch is shown as `BETA_CHANNEL` and the previous release branch + as `STABLE_CHANNEL`: ``` ci/channel-info.sh ``` ### Miscellaneous Clean up -1. Pin the spl-token-cli version in the newly promoted stable branch by setting `splTokenCliVersion` in scripts/spl-token-cli-version.sh to the latest release that depends on the stable branch (usually this will be the latest spl-token-cli release). -1. Update [mergify.yml](https://github.com/anza-xyz/agave/blob/master/.mergify.yml) to add backport actions for the new branch and remove actions for the obsolete branch. -1. Adjust the [Github backport labels](https://github.com/anza-xyz/agave/labels) to add the new branch label and remove the label for the obsolete branch. +1. Pin the spl-token-cli version in the newly promoted stable branch by setting `splTokenCliVersion` in + scripts/spl-token-cli-version.sh to the latest release that depends on the stable branch (usually this will be the + latest spl-token-cli release). +1. Update [mergify.yml](https://github.com/jito-foundation/jito-solana/blob/master/.mergify.yml) to add backport actions + for the new branch and remove actions for the obsolete branch. +1. Adjust the [Github backport labels](https://github.com/jito-foundation/jito-solana/labels) to add the new branch + label and remove the label for the obsolete branch. 1. Announce on Discord #development that the release branch exists so people know to use the new backport labels. ## Steps to Create a Release ### Create the Release Tag on GitHub -1. Go to [GitHub Releases](https://github.com/anza-xyz/agave/releases) for tagging a release. -1. Click "Draft new release". The release tag must exactly match the `version` +1. Go to [GitHub Releases](https://github.com/jito-foundation/jito-solana/releases) for tagging a release. +1. Click "Draft new release". The release tag must exactly match the `version` field in `/Cargo.toml` prefixed by `v`. - 1. If the Cargo.toml version field is **0.12.3**, then the release tag must be **v0.12.3** + 1. If the Cargo.toml version field is **0.12.3**, then the release tag must be **v0.12.3** 1. Make sure the Target Branch field matches the branch you want to make a release on. - 1. If you want to release v0.12.0, the target branch must be v0.12 + 1. If you want to release v0.12.0, the target branch must be v0.12 1. Fill the release notes. - 1. If this is the first release on the branch (e.g. v0.13.**0**), paste in [this - template](https://raw.githubusercontent.com/anza-xyz/agave/master/.github/RELEASE_TEMPLATE.md). Engineering Lead can provide summary contents for release notes if needed. - 1. If this is a patch release, review all the commits since the previous release on this branch and add details as needed. + 1. If this is the first release on the branch (e.g. v0.13.**0**), paste in [this + template](https://raw.githubusercontent.com/jito-foundation/jito-solana/master/.github/RELEASE_TEMPLATE.md). + Engineering Lead can provide summary contents for release notes if needed. + 1. If this is a patch release, review all the commits since the previous release on this branch and add details as + needed. 1. Click "Save Draft", then confirm the release notes look good and the tag name and branch are correct. 1. Ensure all desired commits (usually backports) are landed on the branch by now. -1. Ensure the release is marked **"This is a pre-release"**. This flag will need to be removed manually after confirming the Linux binary artifacts appear at a later step. +1. Ensure the release is marked **"This is a pre-release"**. This flag will need to be removed manually after confirming + the Linux binary artifacts appear at a later step. 1. Go back into edit the release and click "Publish release" while being marked as a pre-release. 1. Confirm there is new git tag with intended version number at the intended revision after running `git fetch` locally. - ### Update release branch with the next patch version -[This action](https://github.com/anza-xyz/agave/blob/master/.github/workflows/increment-cargo-version-on-release.yml) ensures that publishing a release will trigger the creation of a PR to update the Cargo.toml files on **release branch** to the next semantic version (e.g. 0.9.0 -> 0.9.1). Ensure that the created PR makes it through CI and gets submitted. +[This action](https://github.com/jito-foundation/jito-solana/blob/master/.github/workflows/increment-cargo-version-on-release.yml) +ensures that publishing a release will trigger the creation of a PR to update the Cargo.toml files on **release branch** +to the next semantic version (e.g. 0.9.0 -> 0.9.1). Ensure that the created PR makes it through CI and gets submitted. -Note: As of 2024-03-26 the above action is failing so version bumps are done manually. The version bump script is incorrectly updating hashbrown and proc-macro2 versions which should be reverted. +Note: As of 2024-03-26 the above action is failing so version bumps are done manually. The version bump script is +incorrectly updating hashbrown and proc-macro2 versions which should be reverted. ### Prepare for the next release -1. Go to [GitHub Releases](https://github.com/anza-xyz/agave/releases) and create a new draft release for `X.Y.Z+1` with empty release notes. This allows people to incrementally add new release notes until it's time for the next release + +1. Go to [GitHub Releases](https://github.com/jito-foundation/jito-solana/releases) and create a new draft release + for `X.Y.Z+1` with empty release notes. This allows people to incrementally add new release notes until it's time for + the next release 1. Also, point the branch field to the same branch and mark the release as **"This is a pre-release"**. ### Verify release automation success -Go to [Agave Releases](https://github.com/anza-xyz/agave/releases) and click on the latest release that you just published. -Verify that all of the build artifacts are present (15 assets), then uncheck **"This is a pre-release"** for the release. + +Go to [Agave Releases](https://github.com/jito-foundation/jito-solana/releases) and click on the latest release that you +just published. +Verify that all of the build artifacts are present (15 assets), then uncheck **"This is a pre-release"** for the +release. Build artifacts can take up to 60 minutes after creating the tag before -appearing. To check for progress: -* The `agave-secondary` Buildkite pipeline handles creating the Linux and macOS release artifacts and updated crates. Look for a job under the tag name of the release: https://buildkite.com/anza-xyz/agave-secondary. -* The Windows release artifacts are produced by GitHub Actions. Look for a job under the tag name of the release: https://github.com/anza-xyz/agave/actions. +appearing. To check for progress: + +* The `agave-secondary` Buildkite pipeline handles creating the Linux and macOS release artifacts and updated crates. + Look for a job under the tag name of the release: https://buildkite.com/jito-foundation/jito-solana-secondary. +* The Windows release artifacts are produced by GitHub Actions. Look for a job under the tag name of the + release: https://github.com/jito-foundation/jito-solana/actions. -[Crates.io agave-validator](https://crates.io/crates/agave-validator) should have an updated agave-validator version. This can take 2-3 hours, and sometimes fails in the `agave-secondary` job. +[Crates.io agave-validator](https://crates.io/crates/agave-validator) should have an updated agave-validator version. +This can take 2-3 hours, and sometimes fails in the `agave-secondary` job. If this happens and the error is non-fatal, click "Retry" on the "publish crate" job ### Update software on testnet.solana.com -See the documentation at https://github.com/solana-labs/cluster-ops/. devnet.solana.com and mainnet-beta.solana.com run stable releases that have been tested on testnet. Do not update devnet or mainnet-beta with a beta release. + +See the documentation at https://github.com/solana-labs/cluster-ops/. devnet.solana.com and mainnet-beta.solana.com run +stable releases that have been tested on testnet. Do not update devnet or mainnet-beta with a beta release. diff --git a/accounts-db/src/accounts.rs b/accounts-db/src/accounts.rs index b79e8da16c..62e90ef61c 100644 --- a/accounts-db/src/accounts.rs +++ b/accounts-db/src/accounts.rs @@ -540,19 +540,32 @@ impl Accounts { } fn lock_account( - &self, account_locks: &mut AccountLocks, writable_keys: Vec<&Pubkey>, readonly_keys: Vec<&Pubkey>, + additional_read_locks: Option<&HashSet>, + additional_write_locks: Option<&HashSet>, ) -> Result<()> { for k in writable_keys.iter() { - if account_locks.is_locked_write(k) || account_locks.is_locked_readonly(k) { + if account_locks.is_locked_write(k) + || account_locks.is_locked_readonly(k) + || additional_write_locks + .map(|additional_write_locks| additional_write_locks.contains(k)) + .unwrap_or(false) + || additional_read_locks + .map(|additional_read_locks| additional_read_locks.contains(k)) + .unwrap_or(false) + { debug!("Writable account in use: {:?}", k); return Err(TransactionError::AccountInUse); } } for k in readonly_keys.iter() { - if account_locks.is_locked_write(k) { + if account_locks.is_locked_write(k) + || additional_write_locks + .map(|additional_write_locks| additional_write_locks.contains(k)) + .unwrap_or(false) + { debug!("Read-only account in use: {:?}", k); return Err(TransactionError::AccountInUse); } @@ -596,7 +609,7 @@ impl Accounts { let tx_account_locks_results: Vec> = txs .map(|tx| tx.get_account_locks(tx_account_lock_limit)) .collect(); - self.lock_accounts_inner(tx_account_locks_results) + self.lock_accounts_inner(tx_account_locks_results, None, None) } #[must_use] @@ -605,6 +618,8 @@ impl Accounts { txs: impl Iterator, results: impl Iterator>, tx_account_lock_limit: usize, + additional_read_locks: Option<&HashSet>, + additional_write_locks: Option<&HashSet>, ) -> Vec> { let tx_account_locks_results: Vec> = txs .zip(results) @@ -613,22 +628,30 @@ impl Accounts { Err(err) => Err(err), }) .collect(); - self.lock_accounts_inner(tx_account_locks_results) + self.lock_accounts_inner( + tx_account_locks_results, + additional_read_locks, + additional_write_locks, + ) } #[must_use] fn lock_accounts_inner( &self, tx_account_locks_results: Vec>, + additional_read_locks: Option<&HashSet>, + additional_write_locks: Option<&HashSet>, ) -> Vec> { let account_locks = &mut self.account_locks.lock().unwrap(); tx_account_locks_results .into_iter() .map(|tx_account_locks_result| match tx_account_locks_result { - Ok(tx_account_locks) => self.lock_account( + Ok(tx_account_locks) => Self::lock_account( account_locks, tx_account_locks.writable, tx_account_locks.readonly, + additional_read_locks, + additional_write_locks, ), Err(err) => Err(err), }) @@ -673,6 +696,55 @@ impl Accounts { pub fn add_root(&self, slot: Slot) -> AccountsAddRootTiming { self.accounts_db.add_root(slot) } + + pub fn lock_accounts_sequential_with_results<'a>( + &self, + txs: impl Iterator, + tx_account_lock_limit: usize, + ) -> Vec> { + let tx_account_locks_results: Vec> = txs + .map(|tx| tx.get_account_locks(tx_account_lock_limit)) + .collect(); + self.lock_accounts_sequential_inner(tx_account_locks_results) + } + + #[must_use] + fn lock_accounts_sequential_inner( + &self, + tx_account_locks_results: Vec>, + ) -> Vec> { + let mut l_account_locks = self.account_locks.lock().unwrap(); + Self::lock_accounts_sequential(&mut l_account_locks, tx_account_locks_results) + } + + pub fn lock_accounts_sequential( + account_locks: &mut AccountLocks, + tx_account_locks_results: Vec>, + ) -> Vec> { + let mut account_in_use_set = false; + tx_account_locks_results + .into_iter() + .map(|tx_account_locks_result| match tx_account_locks_result { + Ok(tx_account_locks) => match account_in_use_set { + true => Err(TransactionError::AccountInUse), + false => { + let locked = Self::lock_account( + account_locks, + tx_account_locks.writable, + tx_account_locks.readonly, + None, + None, + ); + if matches!(locked, Err(TransactionError::AccountInUse)) { + account_in_use_set = true; + } + locked + } + }, + Err(err) => Err(err), + }) + .collect() + } } #[cfg(test)] @@ -1308,6 +1380,8 @@ mod tests { txs.iter(), qos_results.into_iter(), MAX_TX_ACCOUNT_LOCKS, + None, + None, ); assert_eq!( diff --git a/anchor b/anchor new file mode 160000 index 0000000000..4f52f41cbe --- /dev/null +++ b/anchor @@ -0,0 +1 @@ +Subproject commit 4f52f41cbeafb77d85c7b712516dfbeb5b86dd5f diff --git a/banking-bench/src/main.rs b/banking-bench/src/main.rs index 7d194d044a..6e544039d3 100644 --- a/banking-bench/src/main.rs +++ b/banking-bench/src/main.rs @@ -9,6 +9,7 @@ use { solana_core::{ banking_stage::BankingStage, banking_trace::{BankingPacketBatch, BankingTracer, BANKING_TRACE_DIR_DEFAULT_BYTE_LIMIT}, + bundle_stage::bundle_account_locker::BundleAccountLocker, validator::BlockProductionMethod, }, solana_gossip::cluster_info::{ClusterInfo, Node}, @@ -37,6 +38,7 @@ use { solana_streamer::socket::SocketAddrSpace, solana_tpu_client::tpu_client::DEFAULT_TPU_CONNECTION_POOL_SIZE, std::{ + collections::HashSet, sync::{atomic::Ordering, Arc, RwLock}, thread::sleep, time::{Duration, Instant}, @@ -58,9 +60,15 @@ fn check_txs( let now = Instant::now(); let mut no_bank = false; loop { - if let Ok((_bank, (entry, _tick_height))) = receiver.recv_timeout(Duration::from_millis(10)) + if let Ok(WorkingBankEntry { + bank: _, + entries_ticks, + }) = receiver.recv_timeout(Duration::from_millis(10)) { - total += entry.transactions.len(); + total += entries_ticks + .iter() + .map(|e| e.0.transactions.len()) + .sum::(); } if total >= ref_tx_count { break; @@ -475,6 +483,8 @@ fn main() { bank_forks.clone(), &Arc::new(PrioritizationFeeCache::new(0u64)), false, + HashSet::default(), + BundleAccountLocker::default(), ); // This is so that the signal_receiver does not go out of scope after the closure. diff --git a/banks-server/Cargo.toml b/banks-server/Cargo.toml index 6cf5f77f92..63f6d12b87 100644 --- a/banks-server/Cargo.toml +++ b/banks-server/Cargo.toml @@ -15,6 +15,7 @@ crossbeam-channel = { workspace = true } futures = { workspace = true } solana-banks-interface = { workspace = true } solana-client = { workspace = true } +solana-gossip = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } diff --git a/banks-server/src/banks_server.rs b/banks-server/src/banks_server.rs index c08a41c5d9..3836db9337 100644 --- a/banks-server/src/banks_server.rs +++ b/banks-server/src/banks_server.rs @@ -8,6 +8,7 @@ use { TransactionSimulationDetails, TransactionStatus, }, solana_client::connection_cache::ConnectionCache, + solana_gossip::cluster_info::ClusterInfo, solana_runtime::{ bank::{Bank, TransactionSimulationResult}, bank_forks::BankForks, @@ -425,7 +426,7 @@ pub async fn start_local_server( pub async fn start_tcp_server( listen_addr: SocketAddr, - tpu_addr: SocketAddr, + cluster_info: Arc, bank_forks: Arc>, block_commitment_cache: Arc>, connection_cache: Arc, @@ -450,7 +451,7 @@ pub async fn start_tcp_server( let (sender, receiver) = unbounded(); SendTransactionService::new::( - tpu_addr, + cluster_info.clone(), &bank_forks, None, receiver, diff --git a/bootstrap b/bootstrap new file mode 100755 index 0000000000..d9b1eed6f4 --- /dev/null +++ b/bootstrap @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -eu + +BANK_HASH=$(cargo run --release --bin solana-ledger-tool -- -l config/bootstrap-validator bank-hash) + +# increase max file handle limit +ulimit -Hn 1000000 + +# if above fails, run: +# sudo bash -c 'echo "* hard nofile 1000000" >> /etc/security/limits.conf' + +# NOTE: make sure tip-payment and tip-distribution program are deployed using the correct pubkeys +RUST_LOG=INFO,solana_core::bundle_stage=DEBUG \ + NDEBUG=1 ./multinode-demo/bootstrap-validator.sh \ + --wait-for-supermajority 0 \ + --expected-bank-hash "$BANK_HASH" \ + --block-engine-url http://127.0.0.1 \ + --relayer-url http://127.0.0.1:11226 \ + --rpc-pubsub-enable-block-subscription \ + --enable-rpc-transaction-history \ + --tip-payment-program-pubkey T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt \ + --tip-distribution-program-pubkey 4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7 \ + --commission-bps 0 \ + --shred-receiver-address 127.0.0.1:1002 \ + --trust-relayer-packets \ + --trust-block-engine-packets diff --git a/bundle/Cargo.toml b/bundle/Cargo.toml new file mode 100644 index 0000000000..7b2860e44b --- /dev/null +++ b/bundle/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "solana-bundle" +description = "Library related to handling bundles" +documentation = "https://docs.rs/solana-bundle" +readme = "../README.md" +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +edition = { workspace = true } + +[dependencies] +anchor-lang = { workspace = true } +itertools = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +solana-accounts-db = { workspace = true } +solana-ledger = { workspace = true } +solana-logger = { workspace = true } +solana-measure = { workspace = true } +solana-poh = { workspace = true } +solana-program-runtime = { workspace = true } +solana-runtime = { workspace = true } +solana-sdk = { workspace = true } +solana-svm = { workspace = true } +solana-timings = { workspace = true } +solana-transaction-status = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +assert_matches = { workspace = true } +solana-logger = { workspace = true } +solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } + +[lib] +crate-type = ["lib"] +name = "solana_bundle" diff --git a/bundle/src/bundle_execution.rs b/bundle/src/bundle_execution.rs new file mode 100644 index 0000000000..aa613c39ba --- /dev/null +++ b/bundle/src/bundle_execution.rs @@ -0,0 +1,1216 @@ +use { + itertools::izip, + log::*, + solana_ledger::token_balances::collect_token_balances, + solana_measure::{measure::Measure, measure_us}, + solana_runtime::{ + bank::{Bank, LoadAndExecuteTransactionsOutput, TransactionBalances}, + transaction_batch::TransactionBatch, + }, + solana_sdk::{ + account::AccountSharedData, + bundle::SanitizedBundle, + nonce::state::DurableNonce, + pubkey::Pubkey, + saturating_add_assign, + signature::Signature, + transaction::{SanitizedTransaction, TransactionError, VersionedTransaction}, + }, + solana_svm::{ + account_loader::TransactionLoadResult, + account_overrides::AccountOverrides, + account_saver::collect_accounts_to_store, + transaction_processing_callback::TransactionProcessingCallback, + transaction_processor::{ExecutionRecordingConfig, TransactionProcessingConfig}, + transaction_results::TransactionExecutionResult, + }, + solana_timings::ExecuteTimings, + solana_transaction_status::{token_balances::TransactionTokenBalances, PreBalanceInfo}, + std::{ + cmp::{max, min}, + time::{Duration, Instant}, + }, + thiserror::Error, +}; + +#[derive(Clone, Default)] +pub struct BundleExecutionMetrics { + pub num_retries: u64, + pub collect_balances_us: u64, + pub load_execute_us: u64, + pub collect_pre_post_accounts_us: u64, + pub cache_accounts_us: u64, + pub execute_timings: ExecuteTimings, +} + +/// Contains the results from executing each TransactionBatch with a final result associated with it +/// Note that if !result.is_ok(), bundle_transaction_results will not contain the output for every transaction. +pub struct LoadAndExecuteBundleOutput<'a> { + bundle_transaction_results: Vec>, + result: LoadAndExecuteBundleResult<()>, + metrics: BundleExecutionMetrics, +} + +impl<'a> LoadAndExecuteBundleOutput<'a> { + pub fn executed_ok(&self) -> bool { + self.result.is_ok() + } + + pub fn result(&self) -> &LoadAndExecuteBundleResult<()> { + &self.result + } + + pub fn bundle_transaction_results_mut(&mut self) -> &'a mut [BundleTransactionsOutput] { + &mut self.bundle_transaction_results + } + + pub fn bundle_transaction_results(&self) -> &'a [BundleTransactionsOutput] { + &self.bundle_transaction_results + } + + pub fn executed_transaction_batches(&self) -> Vec> { + self.bundle_transaction_results + .iter() + .map(|br| br.executed_versioned_transactions()) + .collect() + } + + pub fn metrics(&self) -> BundleExecutionMetrics { + self.metrics.clone() + } +} + +#[derive(Clone, Debug, Error)] +pub enum LoadAndExecuteBundleError { + #[error("Bundle execution timed out")] + ProcessingTimeExceeded(Duration), + + #[error( + "A transaction in the bundle encountered a lock error: [signature={:?}, transaction_error={:?}]", + signature, + transaction_error + )] + LockError { + signature: Signature, + transaction_error: TransactionError, + }, + + #[error( + "A transaction in the bundle failed to execute: [signature={:?}, execution_result={:?}", + signature, + execution_result + )] + TransactionError { + signature: Signature, + // Box reduces the size between variants in the Error + execution_result: Box, + }, + + #[error("Invalid pre or post accounts")] + InvalidPreOrPostAccounts, +} + +pub struct BundleTransactionsOutput<'a> { + transactions: &'a [SanitizedTransaction], + load_and_execute_transactions_output: LoadAndExecuteTransactionsOutput, + pre_balance_info: PreBalanceInfo, + post_balance_info: (TransactionBalances, TransactionTokenBalances), + // the length of the outer vector should be the same as transactions.len() + // for indices that didn't get executed, expect a None. + pre_tx_execution_accounts: Vec>>, + post_tx_execution_accounts: Vec>>, +} + +impl<'a> BundleTransactionsOutput<'a> { + pub fn executed_versioned_transactions(&self) -> Vec { + self.transactions + .iter() + .zip( + self.load_and_execute_transactions_output + .execution_results + .iter(), + ) + .filter_map(|(tx, exec_result)| { + exec_result + .was_executed() + .then_some(tx.to_versioned_transaction()) + }) + .collect() + } + + pub fn executed_transactions(&self) -> Vec<&'a SanitizedTransaction> { + self.transactions + .iter() + .zip( + self.load_and_execute_transactions_output + .execution_results + .iter(), + ) + .filter_map(|(tx, exec_result)| exec_result.was_executed().then_some(tx)) + .collect() + } + + pub fn load_and_execute_transactions_output(&self) -> &LoadAndExecuteTransactionsOutput { + &self.load_and_execute_transactions_output + } + + pub fn transactions(&self) -> &[SanitizedTransaction] { + self.transactions + } + + pub fn loaded_transactions_mut(&mut self) -> &mut [TransactionLoadResult] { + &mut self + .load_and_execute_transactions_output + .loaded_transactions + } + + pub fn execution_results(&self) -> &[TransactionExecutionResult] { + &self.load_and_execute_transactions_output.execution_results + } + + pub fn pre_balance_info(&mut self) -> &mut PreBalanceInfo { + &mut self.pre_balance_info + } + + pub fn post_balance_info(&self) -> &(TransactionBalances, TransactionTokenBalances) { + &self.post_balance_info + } + + pub fn pre_tx_execution_accounts(&self) -> &Vec>> { + &self.pre_tx_execution_accounts + } + + pub fn post_tx_execution_accounts(&self) -> &Vec>> { + &self.post_tx_execution_accounts + } +} + +pub type LoadAndExecuteBundleResult = Result; + +/// Return an Error if a transaction was executed and reverted +/// NOTE: `execution_results` are zipped with `sanitized_txs` so it's expected a sanitized tx at +/// position i has a corresponding execution result at position i within the `execution_results` +/// slice +pub fn check_bundle_execution_results<'a>( + execution_results: &'a [TransactionExecutionResult], + sanitized_txs: &'a [SanitizedTransaction], +) -> Result<(), (&'a SanitizedTransaction, &'a TransactionExecutionResult)> { + for (exec_results, sanitized_tx) in execution_results.iter().zip(sanitized_txs) { + match exec_results { + TransactionExecutionResult::Executed { details, .. } => { + if details.status.is_err() { + return Err((sanitized_tx, exec_results)); + } + } + TransactionExecutionResult::NotExecuted(e) => { + if !matches!(e, TransactionError::AccountInUse) { + return Err((sanitized_tx, exec_results)); + } + } + } + } + Ok(()) +} + +/// Executing a bundle is somewhat complicated compared to executing single transactions. In order to +/// avoid duplicate logic for execution and simulation, this function can be leveraged. +/// +/// Assumptions for the caller: +/// - all transactions were signed properly +/// - user has deduplicated transactions inside the bundle +/// +/// TODO (LB): +/// - given a bundle with 3 transactions that write lock the following accounts: [A, B, C], on failure of B +/// we should add in the BundleTransactionsOutput of A and C and return the error for B. +#[allow(clippy::too_many_arguments)] +pub fn load_and_execute_bundle<'a>( + bank: &Bank, + bundle: &'a SanitizedBundle, + // Max blockhash age + max_age: usize, + // Upper bound on execution time for a bundle + max_processing_time: &Duration, + transaction_status_sender_enabled: bool, + log_messages_bytes_limit: &Option, + // simulation will not use the Bank's account locks when building the TransactionBatch + // if simulating on an unfrozen bank, this is helpful to avoid stalling replay and use whatever + // state the accounts are in at the current time + is_simulation: bool, + account_overrides: Option<&mut AccountOverrides>, + // these must be the same length as the bundle's transactions + // allows one to read account state before and after execution of each transaction in the bundle + // will use AccountsOverride + Bank + pre_execution_accounts: &[Option>], + post_execution_accounts: &[Option>], +) -> LoadAndExecuteBundleOutput<'a> { + if pre_execution_accounts.len() != post_execution_accounts.len() + || post_execution_accounts.len() != bundle.transactions.len() + { + return LoadAndExecuteBundleOutput { + bundle_transaction_results: vec![], + result: Err(LoadAndExecuteBundleError::InvalidPreOrPostAccounts), + metrics: BundleExecutionMetrics::default(), + }; + } + + let mut binding = AccountOverrides::default(); + let account_overrides = account_overrides.unwrap_or(&mut binding); + if is_simulation { + bundle + .transactions + .iter() + .map(|tx| tx.message().account_keys()) + .for_each(|account_keys| { + account_overrides.upsert_account_overrides( + bank.get_account_overrides_for_simulation(&account_keys), + ); + + // An unfrozen bank's state is always changing. + // By taking a snapshot of the accounts we're mocking out grabbing their locks. + // **Note** this does not prevent race conditions, just mocks preventing them. + if !bank.is_frozen() { + for pk in account_keys.iter() { + // Save on a disk read. + if account_overrides.get(pk).is_none() { + account_overrides.set_account(pk, bank.get_account_shared_data(pk)); + } + } + } + }); + } + + let mut chunk_start = 0; + let start_time = Instant::now(); + + let mut bundle_transaction_results = vec![]; + let mut metrics = BundleExecutionMetrics::default(); + + while chunk_start != bundle.transactions.len() { + if start_time.elapsed() > *max_processing_time { + trace!("bundle: {} took too long to execute", bundle.bundle_id); + return LoadAndExecuteBundleOutput { + bundle_transaction_results, + metrics, + result: Err(LoadAndExecuteBundleError::ProcessingTimeExceeded( + start_time.elapsed(), + )), + }; + } + + let chunk_end = min(bundle.transactions.len(), chunk_start.saturating_add(128)); + let chunk = &bundle.transactions[chunk_start..chunk_end]; + + // Note: these batches are dropped after execution and before record/commit, which is atypical + // compared to BankingStage which holds account locks until record + commit to avoid race conditions with + // other BankingStage threads. However, the caller of this method, BundleConsumer, will use BundleAccountLocks + // to hold RW locks across all transactions in a bundle until its processed. + let batch = if is_simulation { + bank.prepare_sequential_sanitized_batch_with_results_for_simulation(chunk) + } else { + bank.prepare_sequential_sanitized_batch_with_results(chunk) + }; + + debug!( + "bundle: {} batch num locks ok: {}", + bundle.bundle_id, + batch.lock_results().iter().filter(|lr| lr.is_ok()).count() + ); + + // Ensures that bundle lock results only return either: + // Ok(()) | Err(TransactionError::AccountInUse) + // If the error isn't one of those, then error out + if let Some((transaction, lock_failure)) = batch.check_bundle_lock_results() { + debug!( + "bundle: {} lock error; signature: {} error: {}", + bundle.bundle_id, + transaction.signature(), + lock_failure + ); + return LoadAndExecuteBundleOutput { + bundle_transaction_results, + metrics, + result: Err(LoadAndExecuteBundleError::LockError { + signature: *transaction.signature(), + transaction_error: lock_failure.clone(), + }), + }; + } + + let mut pre_balance_info = PreBalanceInfo::default(); + let (_, collect_balances_us) = measure_us!({ + if transaction_status_sender_enabled { + pre_balance_info.native = + bank.collect_balances_with_cache(&batch, Some(account_overrides)); + pre_balance_info.token = collect_token_balances( + bank, + &batch, + &mut pre_balance_info.mint_decimals, + Some(account_overrides), + ); + } + }); + saturating_add_assign!(metrics.collect_balances_us, collect_balances_us); + + let end = min( + chunk_start.saturating_add(batch.sanitized_transactions().len()), + pre_execution_accounts.len(), + ); + + let m = Measure::start("accounts"); + let accounts_requested = &pre_execution_accounts[chunk_start..end]; + let pre_tx_execution_accounts = + get_account_transactions(bank, account_overrides, accounts_requested, &batch); + saturating_add_assign!(metrics.collect_pre_post_accounts_us, m.end_as_us()); + + let (mut load_and_execute_transactions_output, load_execute_us) = measure_us!(bank + .load_and_execute_transactions( + &batch, + max_age, + &mut metrics.execute_timings, + TransactionProcessingConfig { + account_overrides: Some(account_overrides), + check_program_modification_slot: bank.check_program_modification_slot(), + compute_budget: bank.compute_budget(), + log_messages_bytes_limit: *log_messages_bytes_limit, + limit_to_load_programs: true, + recording_config: ExecutionRecordingConfig::new_single_setting( + transaction_status_sender_enabled + ), + transaction_account_lock_limit: Some(bank.get_transaction_account_lock_limit()), + }, + )); + debug!( + "bundle id: {} loaded_transactions: {:?}", + bundle.bundle_id, load_and_execute_transactions_output.loaded_transactions + ); + saturating_add_assign!(metrics.load_execute_us, load_execute_us); + + // All transactions within a bundle are expected to be executable + not fail + // If there's any transactions that executed and failed or didn't execute due to + // unexpected failures (not locking related), bail out of bundle execution early. + if let Err((failing_tx, exec_result)) = check_bundle_execution_results( + load_and_execute_transactions_output + .execution_results + .as_slice(), + batch.sanitized_transactions(), + ) { + // TODO (LB): we should try to return partial results here for successful bundles in a parallel batch. + // given a bundle that write locks the following accounts [[A], [B], [C]] + // when B fails, we could return the execution results for A and C, but leave B out. + // however, if we have bundle that write locks accounts [[A_1], [A_2], [B], [C]] and B fails + // we'll get the results for A_1 but not [A_2], [B], [C] due to the way this loop executes. + debug!( + "bundle: {} execution error; signature: {} error: {:?}", + bundle.bundle_id, + failing_tx.signature(), + exec_result + ); + return LoadAndExecuteBundleOutput { + bundle_transaction_results, + metrics, + result: Err(LoadAndExecuteBundleError::TransactionError { + signature: *failing_tx.signature(), + execution_result: Box::new(exec_result.clone()), + }), + }; + } + + // If none of the transactions were executed, most likely an AccountInUse error + // need to retry to ensure that all transactions in the bundle are executed. + if !load_and_execute_transactions_output + .execution_results + .iter() + .any(|r| r.was_executed()) + { + saturating_add_assign!(metrics.num_retries, 1); + debug!( + "bundle: {} no transaction executed, retrying", + bundle.bundle_id + ); + continue; + } + + // Cache accounts so next iterations of loop can load cached state instead of using + // AccountsDB, which will contain stale account state because results aren't committed + // to the bank yet. + // NOTE: collect_accounts_to_store does not handle any state changes related to + // failed, non-nonce transactions. + let m = Measure::start("cache"); + + let ((last_blockhash, lamports_per_signature), _last_blockhash_us) = + measure_us!(bank.last_blockhash_and_lamports_per_signature()); + let durable_nonce = DurableNonce::from_blockhash(&last_blockhash); + + let accounts = collect_accounts_to_store( + batch.sanitized_transactions(), + &load_and_execute_transactions_output.execution_results, + &mut load_and_execute_transactions_output.loaded_transactions, + &durable_nonce, + lamports_per_signature, + ) + .0; + for (pubkey, data) in accounts { + account_overrides.set_account(pubkey, Some(data.clone())); + } + saturating_add_assign!(metrics.cache_accounts_us, m.end_as_us()); + + let end = max( + chunk_start.saturating_add(batch.sanitized_transactions().len()), + post_execution_accounts.len(), + ); + + let m = Measure::start("accounts"); + let accounts_requested = &post_execution_accounts[chunk_start..end]; + let post_tx_execution_accounts = + get_account_transactions(bank, account_overrides, accounts_requested, &batch); + saturating_add_assign!(metrics.collect_pre_post_accounts_us, m.end_as_us()); + + let ((post_balances, post_token_balances), collect_balances_us) = + measure_us!(if transaction_status_sender_enabled { + let post_balances = + bank.collect_balances_with_cache(&batch, Some(account_overrides)); + let post_token_balances = collect_token_balances( + bank, + &batch, + &mut pre_balance_info.mint_decimals, + Some(account_overrides), + ); + (post_balances, post_token_balances) + } else { + ( + TransactionBalances::default(), + TransactionTokenBalances::default(), + ) + }); + saturating_add_assign!(metrics.collect_balances_us, collect_balances_us); + + let processing_end = batch.lock_results().iter().position(|lr| lr.is_err()); + if let Some(end) = processing_end { + chunk_start = chunk_start.saturating_add(end); + } else { + chunk_start = chunk_end; + } + + bundle_transaction_results.push(BundleTransactionsOutput { + transactions: chunk, + load_and_execute_transactions_output, + pre_balance_info, + post_balance_info: (post_balances, post_token_balances), + pre_tx_execution_accounts, + post_tx_execution_accounts, + }); + } + + LoadAndExecuteBundleOutput { + bundle_transaction_results, + metrics, + result: Ok(()), + } +} + +fn get_account_transactions( + bank: &Bank, + account_overrides: &AccountOverrides, + accounts: &[Option>], + batch: &TransactionBatch, +) -> Vec>> { + let iter = izip!(batch.lock_results().iter(), accounts.iter()); + + iter.map(|(lock_result, accounts_requested)| { + if lock_result.is_ok() { + accounts_requested.as_ref().map(|accounts_requested| { + accounts_requested + .iter() + .map(|a| match account_overrides.get(a) { + None => (*a, bank.get_account(a).unwrap_or_default()), + Some(data) => (*a, data.clone()), + }) + .collect() + }) + } else { + None + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use { + crate::bundle_execution::{load_and_execute_bundle, LoadAndExecuteBundleError}, + assert_matches::assert_matches, + solana_ledger::genesis_utils::create_genesis_config, + solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::GenesisConfigInfo}, + solana_sdk::{ + bundle::{derive_bundle_id_from_sanitized_transactions, SanitizedBundle}, + clock::MAX_PROCESSING_AGE, + pubkey::Pubkey, + signature::{Keypair, Signer}, + system_transaction::transfer, + transaction::{SanitizedTransaction, Transaction, TransactionError}, + }, + std::{ + sync::{Arc, Barrier, RwLock}, + thread::{sleep, spawn}, + time::Duration, + }, + }; + + const MAX_PROCESSING_TIME: Duration = Duration::from_secs(1); + const LOG_MESSAGE_BYTES_LIMITS: Option = Some(100_000); + const MINT_AMOUNT_LAMPORTS: u64 = 1_000_000; + + fn create_simple_test_bank( + lamports: u64, + ) -> (GenesisConfigInfo, Arc, Arc>) { + let genesis_config_info = create_genesis_config(lamports); + let (bank, bank_forks) = + Bank::new_with_bank_forks_for_tests(&genesis_config_info.genesis_config); + (genesis_config_info, bank, bank_forks) + } + + fn make_bundle(txs: &[Transaction], bank: &Bank) -> SanitizedBundle { + let transactions: Vec<_> = txs + .iter() + .map(|tx| { + SanitizedTransaction::try_from_legacy_transaction( + tx.clone(), + bank.get_reserved_account_keys(), + ) + .unwrap() + }) + .collect(); + + let bundle_id = derive_bundle_id_from_sanitized_transactions(&transactions); + + SanitizedBundle { + transactions, + bundle_id, + } + } + + fn find_account_index(tx: &Transaction, account: &Pubkey) -> Option { + tx.message + .account_keys + .iter() + .position(|pubkey| account == pubkey) + } + + /// A single, valid bundle shall execute successfully and return the correct BundleTransactionsOutput content + #[test] + fn test_single_transaction_bundle_success() { + const TRANSFER_AMOUNT: u64 = 1_000; + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + let lamports_per_signature = bank + .get_lamports_per_signature_for_blockhash(&genesis_config_info.genesis_config.hash()) + .unwrap(); + + let kp = Keypair::new(); + let transactions = vec![transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + TRANSFER_AMOUNT, + genesis_config_info.genesis_config.hash(), + )]; + let bundle = make_bundle(&transactions, &bank); + let default_accounts = vec![None; bundle.transactions.len()]; + + let execution_result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &MAX_PROCESSING_TIME, + true, + &LOG_MESSAGE_BYTES_LIMITS, + false, + None, + &default_accounts, + &default_accounts, + ); + + // make sure the bundle succeeded + assert!(execution_result.result.is_ok()); + + // check to make sure there was one batch returned with one transaction that was the same that was put in + assert_eq!(execution_result.bundle_transaction_results.len(), 1); + let tx_result = execution_result.bundle_transaction_results.first().unwrap(); + assert_eq!(tx_result.transactions.len(), 1); + assert_eq!(tx_result.transactions[0], bundle.transactions[0]); + + // make sure the transaction executed successfully + assert_eq!( + tx_result + .load_and_execute_transactions_output + .execution_results + .len(), + 1 + ); + let execution_result = tx_result + .load_and_execute_transactions_output + .execution_results + .first() + .unwrap(); + assert!(execution_result.was_executed()); + assert!(execution_result.was_executed_successfully()); + + // Make sure the post-balances are correct + assert_eq!(tx_result.pre_balance_info.native.len(), 1); + let post_tx_sol_balances = tx_result.post_balance_info.0.first().unwrap(); + + let minter_message_index = + find_account_index(&transactions[0], &genesis_config_info.mint_keypair.pubkey()) + .unwrap(); + let receiver_message_index = find_account_index(&transactions[0], &kp.pubkey()).unwrap(); + + assert_eq!( + post_tx_sol_balances[minter_message_index], + MINT_AMOUNT_LAMPORTS - lamports_per_signature - TRANSFER_AMOUNT + ); + assert_eq!( + post_tx_sol_balances[receiver_message_index], + TRANSFER_AMOUNT + ); + } + + /// Test a simple failure + #[test] + fn test_single_transaction_bundle_fail() { + const TRANSFER_AMOUNT: u64 = 1_000; + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + + // kp has no funds, transfer will fail + let kp = Keypair::new(); + let transactions = vec![transfer( + &kp, + &kp.pubkey(), + TRANSFER_AMOUNT, + genesis_config_info.genesis_config.hash(), + )]; + let bundle = make_bundle(&transactions, &bank); + + let default_accounts = vec![None; bundle.transactions.len()]; + let execution_result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &MAX_PROCESSING_TIME, + true, + &LOG_MESSAGE_BYTES_LIMITS, + false, + None, + &default_accounts, + &default_accounts, + ); + + assert_eq!(execution_result.bundle_transaction_results.len(), 0); + + assert!(execution_result.result.is_err()); + + match execution_result.result.unwrap_err() { + LoadAndExecuteBundleError::ProcessingTimeExceeded(_) + | LoadAndExecuteBundleError::LockError { .. } + | LoadAndExecuteBundleError::InvalidPreOrPostAccounts => { + unreachable!(); + } + LoadAndExecuteBundleError::TransactionError { + signature, + execution_result, + } => { + assert_eq!(signature, *bundle.transactions[0].signature()); + assert!(!execution_result.was_executed()); + } + } + } + + /// Tests a multi-tx bundle that succeeds. Checks the returned results + #[test] + fn test_multi_transaction_bundle_success() { + const TRANSFER_AMOUNT_1: u64 = 100_000; + const TRANSFER_AMOUNT_2: u64 = 50_000; + const TRANSFER_AMOUNT_3: u64 = 10_000; + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + let lamports_per_signature = bank + .get_lamports_per_signature_for_blockhash(&genesis_config_info.genesis_config.hash()) + .unwrap(); + + // mint transfers 100k to 1 + // 1 transfers 50k to 2 + // 2 transfers 10k to 3 + // should get executed in 3 batches [[1], [2], [3]] + let kp1 = Keypair::new(); + let kp2 = Keypair::new(); + let kp3 = Keypair::new(); + let transactions = vec![ + transfer( + &genesis_config_info.mint_keypair, + &kp1.pubkey(), + TRANSFER_AMOUNT_1, + genesis_config_info.genesis_config.hash(), + ), + transfer( + &kp1, + &kp2.pubkey(), + TRANSFER_AMOUNT_2, + genesis_config_info.genesis_config.hash(), + ), + transfer( + &kp2, + &kp3.pubkey(), + TRANSFER_AMOUNT_3, + genesis_config_info.genesis_config.hash(), + ), + ]; + let bundle = make_bundle(&transactions, &bank); + + let default_accounts = vec![None; bundle.transactions.len()]; + let execution_result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &MAX_PROCESSING_TIME, + true, + &LOG_MESSAGE_BYTES_LIMITS, + false, + None, + &default_accounts, + &default_accounts, + ); + + assert!(execution_result.result.is_ok()); + assert_eq!(execution_result.bundle_transaction_results.len(), 3); + + // first batch contains the first tx that was executed + assert_eq!( + execution_result.bundle_transaction_results[0].transactions, + bundle.transactions + ); + assert_eq!( + execution_result.bundle_transaction_results[0] + .load_and_execute_transactions_output + .execution_results + .len(), + 3 + ); + assert!(execution_result.bundle_transaction_results[0] + .load_and_execute_transactions_output + .execution_results[0] + .was_executed_successfully()); + assert_eq!( + execution_result.bundle_transaction_results[0] + .load_and_execute_transactions_output + .execution_results[1] + .flattened_result(), + Err(TransactionError::AccountInUse) + ); + assert_eq!( + execution_result.bundle_transaction_results[0] + .load_and_execute_transactions_output + .execution_results[2] + .flattened_result(), + Err(TransactionError::AccountInUse) + ); + assert_eq!( + execution_result.bundle_transaction_results[0] + .pre_balance_info + .native + .len(), + 3 + ); + assert_eq!( + execution_result.bundle_transaction_results[0] + .post_balance_info + .0 + .len(), + 3 + ); + + let minter_index = + find_account_index(&transactions[0], &genesis_config_info.mint_keypair.pubkey()) + .unwrap(); + let kp1_index = find_account_index(&transactions[0], &kp1.pubkey()).unwrap(); + + assert_eq!( + execution_result.bundle_transaction_results[0] + .post_balance_info + .0[0][minter_index], + MINT_AMOUNT_LAMPORTS - lamports_per_signature - TRANSFER_AMOUNT_1 + ); + + assert_eq!( + execution_result.bundle_transaction_results[0] + .post_balance_info + .0[0][kp1_index], + TRANSFER_AMOUNT_1 + ); + + // in the second batch, the second transaction was executed + assert_eq!( + execution_result.bundle_transaction_results[1] + .transactions + .to_owned(), + bundle.transactions[1..] + ); + assert_eq!( + execution_result.bundle_transaction_results[1] + .load_and_execute_transactions_output + .execution_results + .len(), + 2 + ); + assert!(execution_result.bundle_transaction_results[1] + .load_and_execute_transactions_output + .execution_results[0] + .was_executed_successfully()); + assert_eq!( + execution_result.bundle_transaction_results[1] + .load_and_execute_transactions_output + .execution_results[1] + .flattened_result(), + Err(TransactionError::AccountInUse) + ); + + assert_eq!( + execution_result.bundle_transaction_results[1] + .pre_balance_info + .native + .len(), + 2 + ); + assert_eq!( + execution_result.bundle_transaction_results[1] + .post_balance_info + .0 + .len(), + 2 + ); + + let kp1_index = find_account_index(&transactions[1], &kp1.pubkey()).unwrap(); + let kp2_index = find_account_index(&transactions[1], &kp2.pubkey()).unwrap(); + + assert_eq!( + execution_result.bundle_transaction_results[1] + .post_balance_info + .0[0][kp1_index], + TRANSFER_AMOUNT_1 - lamports_per_signature - TRANSFER_AMOUNT_2 + ); + + assert_eq!( + execution_result.bundle_transaction_results[1] + .post_balance_info + .0[0][kp2_index], + TRANSFER_AMOUNT_2 + ); + + // in the third batch, the third transaction was executed + assert_eq!( + execution_result.bundle_transaction_results[2] + .transactions + .to_owned(), + bundle.transactions[2..] + ); + assert_eq!( + execution_result.bundle_transaction_results[2] + .load_and_execute_transactions_output + .execution_results + .len(), + 1 + ); + assert!(execution_result.bundle_transaction_results[2] + .load_and_execute_transactions_output + .execution_results[0] + .was_executed_successfully()); + + assert_eq!( + execution_result.bundle_transaction_results[2] + .pre_balance_info + .native + .len(), + 1 + ); + assert_eq!( + execution_result.bundle_transaction_results[2] + .post_balance_info + .0 + .len(), + 1 + ); + + let kp2_index = find_account_index(&transactions[2], &kp2.pubkey()).unwrap(); + let kp3_index = find_account_index(&transactions[2], &kp3.pubkey()).unwrap(); + + assert_eq!( + execution_result.bundle_transaction_results[2] + .post_balance_info + .0[0][kp2_index], + TRANSFER_AMOUNT_2 - lamports_per_signature - TRANSFER_AMOUNT_3 + ); + + assert_eq!( + execution_result.bundle_transaction_results[2] + .post_balance_info + .0[0][kp3_index], + TRANSFER_AMOUNT_3 + ); + } + + /// Tests a multi-tx bundle with the middle transaction failing. + #[test] + fn test_multi_transaction_bundle_fails() { + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + + let kp1 = Keypair::new(); + let kp2 = Keypair::new(); + let kp3 = Keypair::new(); + let transactions = vec![ + transfer( + &genesis_config_info.mint_keypair, + &kp1.pubkey(), + 100_000, + genesis_config_info.genesis_config.hash(), + ), + transfer( + &kp2, + &kp3.pubkey(), + 100_000, + genesis_config_info.genesis_config.hash(), + ), + transfer( + &kp1, + &kp2.pubkey(), + 100_000, + genesis_config_info.genesis_config.hash(), + ), + ]; + let bundle = make_bundle(&transactions, &bank); + + let default_accounts = vec![None; bundle.transactions.len()]; + let execution_result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &MAX_PROCESSING_TIME, + true, + &LOG_MESSAGE_BYTES_LIMITS, + false, + None, + &default_accounts, + &default_accounts, + ); + match execution_result.result.as_ref().unwrap_err() { + LoadAndExecuteBundleError::ProcessingTimeExceeded(_) + | LoadAndExecuteBundleError::LockError { .. } + | LoadAndExecuteBundleError::InvalidPreOrPostAccounts => { + unreachable!(); + } + + LoadAndExecuteBundleError::TransactionError { + signature, + execution_result: tx_failure, + } => { + assert_eq!(signature, bundle.transactions[1].signature()); + assert_eq!( + tx_failure.flattened_result(), + Err(TransactionError::AccountNotFound) + ); + assert_eq!(execution_result.bundle_transaction_results().len(), 0); + } + } + } + + /// Tests that when the max processing time is exceeded, the bundle is an error + #[test] + fn test_bundle_max_processing_time_exceeded() { + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + + let kp = Keypair::new(); + let transactions = vec![transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + 1, + genesis_config_info.genesis_config.hash(), + )]; + let bundle = make_bundle(&transactions, &bank); + + let locked_transfer = vec![SanitizedTransaction::from_transaction_for_tests(transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + 2, + genesis_config_info.genesis_config.hash(), + ))]; + + // locks it and prevents execution bc write lock on genesis_config_info.mint_keypair + kp.pubkey() held + let _batch = bank.prepare_sanitized_batch(&locked_transfer); + + let default = vec![None; bundle.transactions.len()]; + let result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &Duration::from_millis(100), + false, + &None, + false, + None, + &default, + &default, + ); + assert_matches!( + result.result, + Err(LoadAndExecuteBundleError::ProcessingTimeExceeded(_)) + ); + } + + #[test] + fn test_simulate_bundle_with_locked_account_works() { + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + + let kp = Keypair::new(); + let transactions = vec![transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + 1, + genesis_config_info.genesis_config.hash(), + )]; + let bundle = make_bundle(&transactions, &bank); + + let locked_transfer = vec![SanitizedTransaction::from_transaction_for_tests(transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + 2, + genesis_config_info.genesis_config.hash(), + ))]; + + let _batch = bank.prepare_sanitized_batch(&locked_transfer); + + // simulation ignores account locks so you can simulate bundles on unfrozen banks + let default = vec![None; bundle.transactions.len()]; + let result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &Duration::from_millis(100), + false, + &None, + true, + None, + &default, + &default, + ); + assert!(result.result.is_ok()); + } + + /// Creates a multi-tx bundle and temporarily locks the accounts for one of the transactions in a bundle. + /// Ensures the result is what's expected + #[test] + fn test_bundle_works_with_released_account_locks() { + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + let barrier = Arc::new(Barrier::new(2)); + + let kp = Keypair::new(); + + let transactions = vec![transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + 1, + genesis_config_info.genesis_config.hash(), + )]; + let bundle = make_bundle(&transactions, &bank); + + let locked_transfer = vec![SanitizedTransaction::from_transaction_for_tests(transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + 2, + genesis_config_info.genesis_config.hash(), + ))]; + + // background thread locks the accounts for a bit then unlocks them + let thread = { + let barrier = barrier.clone(); + let bank = bank.clone(); + spawn(move || { + let batch = bank.prepare_sanitized_batch(&locked_transfer); + barrier.wait(); + sleep(Duration::from_millis(500)); + drop(batch); + }) + }; + + let _ = barrier.wait(); + + // load_and_execute_bundle should spin for a bit then process after the 500ms sleep is over + let default = vec![None; bundle.transactions.len()]; + let result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &Duration::from_secs(2), + false, + &None, + false, + None, + &default, + &default, + ); + assert!(result.result.is_ok()); + + thread.join().unwrap(); + } + + /// Tests that when the max processing time is exceeded, the bundle is an error + #[test] + fn test_bundle_bad_pre_post_accounts() { + const PRE_EXECUTION_ACCOUNTS: [Option>; 2] = [None, None]; + let (genesis_config_info, bank, _bank_forks) = + create_simple_test_bank(MINT_AMOUNT_LAMPORTS); + + let kp = Keypair::new(); + let transactions = vec![transfer( + &genesis_config_info.mint_keypair, + &kp.pubkey(), + 1, + genesis_config_info.genesis_config.hash(), + )]; + let bundle = make_bundle(&transactions, &bank); + + let result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &Duration::from_millis(100), + false, + &None, + false, + None, + &PRE_EXECUTION_ACCOUNTS, + &vec![None; bundle.transactions.len()], + ); + assert_matches!( + result.result, + Err(LoadAndExecuteBundleError::InvalidPreOrPostAccounts) + ); + + let result = load_and_execute_bundle( + &bank, + &bundle, + MAX_PROCESSING_AGE, + &Duration::from_millis(100), + false, + &None, + false, + None, + &vec![None; bundle.transactions.len()], + &PRE_EXECUTION_ACCOUNTS, + ); + assert_matches!( + result.result, + Err(LoadAndExecuteBundleError::InvalidPreOrPostAccounts) + ); + } +} diff --git a/bundle/src/lib.rs b/bundle/src/lib.rs new file mode 100644 index 0000000000..a93e0d3d17 --- /dev/null +++ b/bundle/src/lib.rs @@ -0,0 +1,60 @@ +use { + crate::bundle_execution::LoadAndExecuteBundleError, + anchor_lang::error::Error, + serde::{Deserialize, Serialize}, + solana_poh::poh_recorder::PohRecorderError, + solana_sdk::pubkey::Pubkey, + thiserror::Error, +}; + +pub mod bundle_execution; + +#[derive(Error, Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum TipError { + #[error("account is missing from bank: {0}")] + AccountMissing(Pubkey), + + #[error("Anchor error: {0}")] + AnchorError(String), + + #[error("Lock error")] + LockError, + + #[error("Error executing initialize programs")] + InitializeProgramsError, + + #[error("Error cranking tip programs")] + CrankTipError, +} + +impl From for TipError { + fn from(anchor_err: Error) -> Self { + match anchor_err { + Error::AnchorError(e) => Self::AnchorError(e.error_msg), + Error::ProgramError(e) => Self::AnchorError(e.to_string()), + } + } +} + +pub type BundleExecutionResult = Result; + +#[derive(Error, Debug, Clone)] +pub enum BundleExecutionError { + #[error("The bank has hit the max allotted time for processing transactions")] + BankProcessingTimeLimitReached, + + #[error("The bundle exceeds the cost model")] + ExceedsCostModel, + + #[error("Runtime error while executing the bundle: {0}")] + TransactionFailure(#[from] LoadAndExecuteBundleError), + + #[error("Error locking bundle because a transaction is malformed")] + LockError, + + #[error("PoH record error: {0}")] + PohRecordError(#[from] PohRecorderError), + + #[error("Tip payment error {0}")] + TipError(#[from] TipError), +} diff --git a/ci/buildkite-pipeline-in-disk.sh b/ci/buildkite-pipeline-in-disk.sh index 81b084ea75..c5877ee6f5 100755 --- a/ci/buildkite-pipeline-in-disk.sh +++ b/ci/buildkite-pipeline-in-disk.sh @@ -289,7 +289,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" + "https://github.com/jito-foundation/jito-solana/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -307,7 +307,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/jito-foundation/jito-solana/$BUILDKITE_BRANCH" pull_or_push_steps exit 0 diff --git a/ci/buildkite-pipeline.sh b/ci/buildkite-pipeline.sh index 4d1559b606..8e2d235cb1 100755 --- a/ci/buildkite-pipeline.sh +++ b/ci/buildkite-pipeline.sh @@ -353,7 +353,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" + "https://github.com/jito-foundation/jito-solana/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -371,7 +371,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/jito-foundation/jito-solana/$BUILDKITE_BRANCH" pull_or_push_steps exit 0 diff --git a/ci/buildkite-secondary.yml b/ci/buildkite-secondary.yml index c43c7ee449..627a73b2c2 100644 --- a/ci/buildkite-secondary.yml +++ b/ci/buildkite-secondary.yml @@ -18,34 +18,34 @@ steps: agents: queue: "release-build" timeout_in_minutes: 5 - - wait - - name: "publish docker" - command: "sdk/docker-solana/build.sh" - agents: - queue: "release-build" - timeout_in_minutes: 60 - - name: "publish crate" - command: "ci/publish-crate.sh" - agents: - queue: "release-build" - retry: - manual: - permit_on_passed: true - timeout_in_minutes: 240 - branches: "!master" - - name: "publish tarball (aarch64-apple-darwin)" - command: "ci/publish-tarball.sh" - agents: - queue: "release-build-aarch64-apple-darwin" - retry: - manual: - permit_on_passed: true - timeout_in_minutes: 60 - - name: "publish tarball (x86_64-apple-darwin)" - command: "ci/publish-tarball.sh" - agents: - queue: "release-build-x86_64-apple-darwin" - retry: - manual: - permit_on_passed: true - timeout_in_minutes: 60 +# - wait +# - name: "publish docker" +# command: "sdk/docker-solana/build.sh" +# agents: +# queue: "release-build" +# timeout_in_minutes: 60 +# - name: "publish crate" +# command: "ci/publish-crate.sh" +# agents: +# queue: "release-build" +# retry: +# manual: +# permit_on_passed: true +# timeout_in_minutes: 240 +# branches: "!master" +# - name: "publish tarball (aarch64-apple-darwin)" +# command: "ci/publish-tarball.sh" +# agents: +# queue: "release-build-aarch64-apple-darwin" +# retry: +# manual: +# permit_on_passed: true +# timeout_in_minutes: 60 +# - name: "publish tarball (x86_64-apple-darwin)" +# command: "ci/publish-tarball.sh" +# agents: +# queue: "release-build-x86_64-apple-darwin" +# retry: +# manual: +# permit_on_passed: true +# timeout_in_minutes: 60 diff --git a/ci/buildkite-solana-private.sh b/ci/buildkite-solana-private.sh index d514ac0ad2..5bbf5ca034 100755 --- a/ci/buildkite-solana-private.sh +++ b/ci/buildkite-solana-private.sh @@ -269,7 +269,7 @@ pull_or_push_steps() { # start_pipeline "Tag pipeline for $BUILDKITE_TAG" # annotate --style info --context release-tag \ -# "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG" +# "https://github.com/jito-foundation/jito-solana/releases/$BUILDKITE_TAG" # # Jump directly to the secondary build to publish release artifacts quickly # trigger_secondary_step @@ -287,7 +287,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/jito-foundation/jito-solana/$BUILDKITE_BRANCH" pull_or_push_steps exit 0 diff --git a/ci/channel-info.sh b/ci/channel-info.sh index 2bb8083656..101583307f 100755 --- a/ci/channel-info.sh +++ b/ci/channel-info.sh @@ -11,7 +11,7 @@ here="$(dirname "$0")" # shellcheck source=ci/semver_bash/semver.sh source "$here"/semver_bash/semver.sh -remote=https://github.com/anza-xyz/agave.git +remote=https://github.com/jito-foundation/jito-solana.git # Fetch all vX.Y.Z tags # diff --git a/ci/check-crates.sh b/ci/check-crates.sh index 4f407824a3..802fe7b63c 100755 --- a/ci/check-crates.sh +++ b/ci/check-crates.sh @@ -31,6 +31,9 @@ printf "%s\n" "${files[@]}" error_count=0 for file in "${files[@]}"; do read -r crate_name package_publish workspace < <(toml get "$file" . | jq -r '(.package.name | tostring)+" "+(.package.publish | tostring)+" "+(.workspace | tostring)') + if [ "$crate_name" == "solana-bundle" ]; then + continue + fi echo "=== $crate_name ($file) ===" if [[ $package_publish = 'false' ]]; then diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh index f7d98ffd5d..b702e70285 100755 --- a/ci/publish-installer.sh +++ b/ci/publish-installer.sh @@ -26,14 +26,15 @@ fi # upload install script source ci/upload-ci-artifact.sh -cat >release.anza.xyz-install <release.jito.wtf-install <>release.anza.xyz-install +cat install/agave-install-init.sh >>release.jito.wtf-install echo --- GCS: "install" -upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install" +upload-gcs-artifact "/solana/release.jito.wtf-install" "gs://jito-release/$CHANNEL_OR_TAG/install" echo Published to: -ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/install +ci/format-url.sh https://release.jito.wtf/"$CHANNEL_OR_TAG"/install + diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh index 4746e9fb2f..9401f74915 100755 --- a/ci/publish-tarball.sh +++ b/ci/publish-tarball.sh @@ -119,10 +119,10 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET. if [[ -n $BUILDKITE ]]; then echo --- GCS Store: "$file" - upload-gcs-artifact "/solana/$file" gs://anza-release/"$CHANNEL_OR_TAG"/"$file" + upload-gcs-artifact "/solana/$file" gs://jito-release/"$CHANNEL_OR_TAG"/"$file" echo Published to: - $DRYRUN ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/"$file" + $DRYRUN ci/format-url.sh https://release.jito.wtf/"$CHANNEL_OR_TAG"/"$file" if [[ -n $TAG ]]; then ci/upload-github-release-asset.sh "$file" diff --git a/ci/test-coverage.sh b/ci/test-coverage.sh index f4288285a4..323241b294 100755 --- a/ci/test-coverage.sh +++ b/ci/test-coverage.sh @@ -40,5 +40,5 @@ else codecov -t "${CODECOV_TOKEN}" --dir "$here/../target/cov/${SHORT_CI_COMMIT}" annotate --style success --context codecov.io \ - "CodeCov report: https://codecov.io/github/anza-xyz/agave/commit/$CI_COMMIT" + "CodeCov report: https://codecov.io/github/jito-foundation/jito-solana/commit/$CI_COMMIT" fi diff --git a/ci/upload-github-release-asset.sh b/ci/upload-github-release-asset.sh index 229fb8993e..fb4de1af9e 100755 --- a/ci/upload-github-release-asset.sh +++ b/ci/upload-github-release-asset.sh @@ -26,7 +26,7 @@ fi # Force CI_REPO_SLUG since sometimes # BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG is not set correctly, causing the # artifact upload to fail -CI_REPO_SLUG=anza-xyz/agave +CI_REPO_SLUG=jito-foundation/jito-solana #if [[ -z $CI_REPO_SLUG ]]; then # echo Error: CI_REPO_SLUG not defined # exit 1 diff --git a/core/Cargo.toml b/core/Cargo.toml index f0a2d17871..251a75d0e8 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -15,6 +15,7 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git [dependencies] ahash = { workspace = true } +anchor-lang = { workspace = true } arrayvec = { workspace = true } base64 = { workspace = true } bincode = { workspace = true } @@ -27,12 +28,17 @@ etcd-client = { workspace = true, features = ["tls"] } futures = { workspace = true } histogram = { workspace = true } itertools = { workspace = true } +jito-protos = { workspace = true } +jito-tip-distribution = { workspace = true } +jito-tip-payment = { workspace = true } lazy_static = { workspace = true } log = { workspace = true } lru = { workspace = true } min-max-heap = { workspace = true } num_enum = { workspace = true } prio-graph = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } qualifier_attr = { workspace = true } quinn = { workspace = true } rand = { workspace = true } @@ -45,6 +51,7 @@ serde_bytes = { workspace = true } serde_derive = { workspace = true } solana-accounts-db = { workspace = true } solana-bloom = { workspace = true } +solana-bundle = { workspace = true } solana-client = { workspace = true } solana-compute-budget = { workspace = true } solana-connection-cache = { workspace = true } @@ -65,6 +72,7 @@ solana-rayon-threadlimit = { workspace = true } solana-rpc = { workspace = true } solana-rpc-client-api = { workspace = true } solana-runtime = { workspace = true } +solana-runtime-plugin = { workspace = true } solana-sanitize = { workspace = true } solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } @@ -86,6 +94,7 @@ sys-info = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } +tonic = { workspace = true } trees = { workspace = true } [dev-dependencies] @@ -93,12 +102,15 @@ assert_matches = { workspace = true } fs_extra = { workspace = true } serde_json = { workspace = true } serial_test = { workspace = true } +solana-accounts-db = { workspace = true } # See order-crates-for-publishing.py for using this unusual `path = "."` +solana-bundle = { workspace = true } solana-core = { path = ".", features = ["dev-context-only-utils"] } solana-ledger = { workspace = true, features = ["dev-context-only-utils"] } solana-logger = { workspace = true } solana-poh = { workspace = true, features = ["dev-context-only-utils"] } solana-program-runtime = { workspace = true } +solana-program-test = { workspace = true } solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } solana-sdk = { workspace = true, features = ["dev-context-only-utils"] } solana-stake-program = { workspace = true } @@ -114,6 +126,7 @@ sysctl = { workspace = true } [build-dependencies] rustc_version = { workspace = true } +tonic-build = { workspace = true } [features] dev-context-only-utils = [] diff --git a/core/benches/banking_stage.rs b/core/benches/banking_stage.rs index d0efbfafdd..95d72167da 100644 --- a/core/benches/banking_stage.rs +++ b/core/benches/banking_stage.rs @@ -25,6 +25,7 @@ use { BankingStage, BankingStageStats, }, banking_trace::{BankingPacketBatch, BankingTracer}, + bundle_stage::bundle_account_locker::BundleAccountLocker, }, solana_entry::entry::{next_hash, Entry}, solana_gossip::cluster_info::{ClusterInfo, Node}, @@ -54,6 +55,7 @@ use { }, solana_streamer::socket::SocketAddrSpace, std::{ + collections::HashSet, iter::repeat_with, sync::{atomic::Ordering, Arc}, time::{Duration, Instant}, @@ -65,8 +67,15 @@ fn check_txs(receiver: &Arc>, ref_tx_count: usize) { let mut total = 0; let now = Instant::now(); loop { - if let Ok((_bank, (entry, _tick_height))) = receiver.recv_timeout(Duration::new(1, 0)) { - total += entry.transactions.len(); + if let Ok(WorkingBankEntry { + bank: _, + entries_ticks, + }) = receiver.recv_timeout(Duration::new(1, 0)) + { + total += entries_ticks + .iter() + .map(|e| e.0.transactions.len()) + .sum::(); } if total >= ref_tx_count { break; @@ -110,7 +119,14 @@ fn bench_consume_buffered(bencher: &mut Bencher) { ); let (s, _r) = unbounded(); let committer = Committer::new(None, s, Arc::new(PrioritizationFeeCache::new(0u64))); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); // This tests the performance of buffering packets. // If the packet buffers are copied, performance will be poor. bencher.iter(move || { @@ -304,6 +320,8 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) { bank_forks, &Arc::new(PrioritizationFeeCache::new(0u64)), false, + HashSet::default(), + BundleAccountLocker::default(), ); let chunk_len = verified.len() / CHUNKS; diff --git a/core/benches/consumer.rs b/core/benches/consumer.rs index d736b93ef9..d0486219b0 100644 --- a/core/benches/consumer.rs +++ b/core/benches/consumer.rs @@ -7,16 +7,16 @@ use { iter::IndexedParallelIterator, prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, }, - solana_core::banking_stage::{ - committer::Committer, consumer::Consumer, qos_service::QosService, + solana_core::{ + banking_stage::{committer::Committer, consumer::Consumer, qos_service::QosService}, + bundle_stage::bundle_account_locker::BundleAccountLocker, }, - solana_entry::entry::Entry, solana_ledger::{ blockstore::Blockstore, genesis_utils::{create_genesis_config, GenesisConfigInfo}, }, solana_poh::{ - poh_recorder::{create_test_recorder, PohRecorder}, + poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry}, poh_service::PohService, }, solana_runtime::{bank::Bank, bank_forks::BankForks}, @@ -28,9 +28,12 @@ use { system_program, system_transaction, transaction::SanitizedTransaction, }, - std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, RwLock, + std::{ + collections::HashSet, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, RwLock, + }, }, tempfile::TempDir, test::Bencher, @@ -84,7 +87,14 @@ fn create_consumer(poh_recorder: &RwLock) -> Consumer { let (replay_vote_sender, _replay_vote_receiver) = unbounded(); let committer = Committer::new(None, replay_vote_sender, Arc::default()); let transaction_recorder = poh_recorder.read().unwrap().new_recorder(); - Consumer::new(committer, transaction_recorder, QosService::new(0), None) + Consumer::new( + committer, + transaction_recorder, + QosService::new(0), + None, + HashSet::default(), + BundleAccountLocker::default(), + ) } struct BenchFrame { @@ -94,7 +104,7 @@ struct BenchFrame { exit: Arc, poh_recorder: Arc>, poh_service: PohService, - signal_receiver: Receiver<(Arc, (Entry, u64))>, + signal_receiver: Receiver, } fn setup() -> BenchFrame { diff --git a/core/benches/proto_to_packet.rs b/core/benches/proto_to_packet.rs new file mode 100644 index 0000000000..87f85f9c7f --- /dev/null +++ b/core/benches/proto_to_packet.rs @@ -0,0 +1,56 @@ +#![feature(test)] + +extern crate test; + +use { + jito_protos::proto::packet::{ + Meta as PbMeta, Packet as PbPacket, PacketBatch, PacketFlags as PbFlags, + }, + solana_core::proto_packet_to_packet, + solana_sdk::packet::{Packet, PACKET_DATA_SIZE}, + std::iter::repeat, + test::{black_box, Bencher}, +}; + +fn get_proto_packet(i: u8) -> PbPacket { + PbPacket { + data: repeat(i).take(PACKET_DATA_SIZE).collect(), + meta: Some(PbMeta { + size: PACKET_DATA_SIZE as u64, + addr: "255.255.255.255:65535".to_string(), + port: 65535, + flags: Some(PbFlags { + discard: false, + forwarded: false, + repair: false, + simple_vote_tx: false, + tracer_packet: false, + }), + sender_stake: 0, + }), + } +} + +#[bench] +fn bench_proto_to_packet(bencher: &mut Bencher) { + bencher.iter(|| { + black_box(proto_packet_to_packet(get_proto_packet(1))); + }); +} + +#[bench] +fn bench_batch_list_to_packets(bencher: &mut Bencher) { + let packet_batch = PacketBatch { + packets: (0..128).map(get_proto_packet).collect(), + }; + + bencher.iter(|| { + black_box( + packet_batch + .packets + .iter() + .map(|p| proto_packet_to_packet(p.clone())) + .collect::>(), + ); + }); +} diff --git a/core/src/admin_rpc_post_init.rs b/core/src/admin_rpc_post_init.rs index 364509a63b..425a4375c1 100644 --- a/core/src/admin_rpc_post_init.rs +++ b/core/src/admin_rpc_post_init.rs @@ -1,6 +1,7 @@ use { crate::{ cluster_slots_service::cluster_slots::ClusterSlots, + proxy::{block_engine_stage::BlockEngineConfig, relayer_stage::RelayerConfig}, repair::{outstanding_requests::OutstandingRequests, serve_repair::ShredRepairType}, }, solana_gossip::cluster_info::ClusterInfo, @@ -8,8 +9,8 @@ use { solana_sdk::{pubkey::Pubkey, quic::NotifyKeyUpdate}, std::{ collections::HashSet, - net::UdpSocket, - sync::{Arc, RwLock}, + net::{SocketAddr, UdpSocket}, + sync::{Arc, Mutex, RwLock}, }, }; @@ -23,4 +24,7 @@ pub struct AdminRpcRequestMetadataPostInit { pub repair_socket: Arc, pub outstanding_repair_requests: Arc>>, pub cluster_slots: Arc, + pub block_engine_config: Arc>, + pub relayer_config: Arc>, + pub shred_receiver_address: Arc>>, } diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 1bdd7defee..607b7ee10d 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -25,6 +25,7 @@ use { }, }, banking_trace::BankingPacketReceiver, + bundle_stage::bundle_account_locker::BundleAccountLocker, tracer_packet_stats::TracerPacketStats, validator::BlockProductionMethod, }, @@ -40,9 +41,11 @@ use { bank_forks::BankForks, prioritization_fee_cache::PrioritizationFeeCache, vote_sender_types::ReplayVoteSender, }, - solana_sdk::timing::AtomicInterval, + solana_sdk::{pubkey::Pubkey, timing::AtomicInterval}, std::{ - cmp, env, + cmp, + collections::HashSet, + env, sync::{ atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, RwLock, @@ -62,12 +65,12 @@ pub mod unprocessed_packet_batches; pub mod unprocessed_transaction_storage; mod consume_worker; -mod decision_maker; +pub(crate) mod decision_maker; mod forward_packet_batches_by_accounts; mod forward_worker; -mod immutable_deserialized_packet; +pub(crate) mod immutable_deserialized_packet; mod latest_unprocessed_votes; -mod leader_slot_timing_metrics; +pub(crate) mod leader_slot_timing_metrics; mod multi_iterator_scanner; mod packet_deserializer; mod packet_filter; @@ -340,6 +343,8 @@ impl BankingStage { bank_forks: Arc>, prioritization_fee_cache: &Arc, enable_forwarding: bool, + blacklisted_accounts: HashSet, + bundle_account_locker: BundleAccountLocker, ) -> Self { Self::new_num_threads( block_production_method, @@ -356,6 +361,8 @@ impl BankingStage { bank_forks, prioritization_fee_cache, enable_forwarding, + blacklisted_accounts, + bundle_account_locker, ) } @@ -375,6 +382,8 @@ impl BankingStage { bank_forks: Arc>, prioritization_fee_cache: &Arc, enable_forwarding: bool, + blacklisted_accounts: HashSet, + bundle_account_locker: BundleAccountLocker, ) -> Self { match block_production_method { BlockProductionMethod::ThreadLocalMultiIterator => { @@ -391,6 +400,8 @@ impl BankingStage { connection_cache, bank_forks, prioritization_fee_cache, + blacklisted_accounts, + bundle_account_locker, ) } BlockProductionMethod::CentralScheduler => Self::new_central_scheduler( @@ -407,6 +418,8 @@ impl BankingStage { bank_forks, prioritization_fee_cache, enable_forwarding, + blacklisted_accounts, + bundle_account_locker, ), } } @@ -425,6 +438,8 @@ impl BankingStage { connection_cache: Arc, bank_forks: Arc>, prioritization_fee_cache: &Arc, + blacklisted_accounts: HashSet, + bundle_account_locker: BundleAccountLocker, ) -> Self { assert!(num_threads >= MIN_TOTAL_THREADS); // Single thread to generate entries from many banks. @@ -489,6 +504,8 @@ impl BankingStage { log_messages_bytes_limit, forwarder, unprocessed_transaction_storage, + blacklisted_accounts.clone(), + bundle_account_locker.clone(), ) }) .collect(); @@ -510,6 +527,8 @@ impl BankingStage { bank_forks: Arc>, prioritization_fee_cache: &Arc, enable_forwarding: bool, + blacklisted_accounts: HashSet, + bundle_account_locker: BundleAccountLocker, ) -> Self { assert!(num_threads >= MIN_TOTAL_THREADS); // Single thread to generate entries from many banks. @@ -554,6 +573,8 @@ impl BankingStage { latest_unprocessed_votes.clone(), vote_source, ), + blacklisted_accounts.clone(), + bundle_account_locker.clone(), )); } @@ -575,6 +596,8 @@ impl BankingStage { poh_recorder.read().unwrap().new_recorder(), QosService::new(id), log_messages_bytes_limit, + blacklisted_accounts.clone(), + bundle_account_locker.clone(), ), finished_work_sender.clone(), poh_recorder.read().unwrap().new_leader_bank_notifier(), @@ -629,6 +652,7 @@ impl BankingStage { Self { bank_thread_hdls } } + #[allow(clippy::too_many_arguments)] fn spawn_thread_local_multi_iterator_thread( id: u32, packet_receiver: BankingPacketReceiver, @@ -639,13 +663,18 @@ impl BankingStage { log_messages_bytes_limit: Option, mut forwarder: Forwarder, unprocessed_transaction_storage: UnprocessedTransactionStorage, + blacklisted_accounts: HashSet, + bundle_account_locker: BundleAccountLocker, ) -> JoinHandle<()> { let mut packet_receiver = PacketReceiver::new(id, packet_receiver, bank_forks); + let consumer = Consumer::new( committer, transaction_recorder, QosService::new(id), log_messages_bytes_limit, + blacklisted_accounts.clone(), + bundle_account_locker.clone(), ); Builder::new() @@ -806,7 +835,7 @@ mod tests { crate::banking_trace::{BankingPacketBatch, BankingTracer}, crossbeam_channel::{unbounded, Receiver}, itertools::Itertools, - solana_entry::entry::{self, Entry, EntrySlice}, + solana_entry::entry::{self, EntrySlice}, solana_gossip::cluster_info::Node, solana_ledger::{ blockstore::Blockstore, @@ -820,6 +849,7 @@ mod tests { solana_poh::{ poh_recorder::{ create_test_recorder, PohRecorderError, Record, RecordTransactionsSummary, + WorkingBankEntry, }, poh_service::PohService, }, @@ -891,6 +921,8 @@ mod tests { bank_forks, &Arc::new(PrioritizationFeeCache::new(0u64)), false, + HashSet::default(), + BundleAccountLocker::default(), ); drop(non_vote_sender); drop(tpu_vote_sender); @@ -947,6 +979,8 @@ mod tests { bank_forks, &Arc::new(PrioritizationFeeCache::new(0u64)), false, + HashSet::default(), + BundleAccountLocker::default(), ); trace!("sending bank"); drop(non_vote_sender); @@ -959,7 +993,12 @@ mod tests { trace!("getting entries"); let entries: Vec<_> = entry_receiver .iter() - .map(|(_bank, (entry, _tick_height))| entry) + .flat_map( + |WorkingBankEntry { + bank: _, + entries_ticks, + }| entries_ticks.into_iter().map(|(e, _)| e), + ) .collect(); trace!("done"); assert_eq!(entries.len(), genesis_config.ticks_per_slot as usize); @@ -1027,6 +1066,8 @@ mod tests { bank_forks.clone(), // keep a local-copy of bank-forks so worker threads do not lose weak access to bank-forks &Arc::new(PrioritizationFeeCache::new(0u64)), false, + HashSet::default(), + BundleAccountLocker::default(), ); // fund another account so we can send 2 good transactions in a single batch. @@ -1078,9 +1119,14 @@ mod tests { bank.process_transaction(&fund_tx).unwrap(); //receive entries + ticks loop { - let entries: Vec = entry_receiver + let entries: Vec<_> = entry_receiver .iter() - .map(|(_bank, (entry, _tick_height))| entry) + .flat_map( + |WorkingBankEntry { + bank: _, + entries_ticks, + }| entries_ticks.into_iter().map(|(e, _)| e), + ) .collect(); assert!(entries.verify(&blockhash, &entry::thread_pool_for_tests())); @@ -1197,6 +1243,8 @@ mod tests { Arc::new(ConnectionCache::new("connection_cache_test")), bank_forks, &Arc::new(PrioritizationFeeCache::new(0u64)), + HashSet::default(), + BundleAccountLocker::default(), ); // wait for banking_stage to eat the packets @@ -1215,7 +1263,12 @@ mod tests { // check that the balance is what we expect. let entries: Vec<_> = entry_receiver .iter() - .map(|(_bank, (entry, _tick_height))| entry) + .flat_map( + |WorkingBankEntry { + bank: _, + entries_ticks, + }| entries_ticks.into_iter().map(|(e, _)| e), + ) .collect(); let (bank, _bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); @@ -1278,15 +1331,19 @@ mod tests { system_transaction::transfer(&keypair2, &pubkey2, 1, genesis_config.hash()).into(), ]; - let _ = recorder.record_transactions(bank.slot(), txs.clone()); - let (_bank, (entry, _tick_height)) = entry_receiver.recv().unwrap(); + let _ = recorder.record_transactions(bank.slot(), vec![txs.clone()]); + let WorkingBankEntry { + bank, + entries_ticks, + } = entry_receiver.recv().unwrap(); + let entry = &entries_ticks.first().unwrap().0; assert_eq!(entry.transactions, txs); // Once bank is set to a new bank (setting bank.slot() + 1 in record_transactions), // record_transactions should throw MaxHeightReached let next_slot = bank.slot() + 1; let RecordTransactionsSummary { result, .. } = - recorder.record_transactions(next_slot, txs); + recorder.record_transactions(next_slot, vec![txs]); assert_matches!(result, Err(PohRecorderError::MaxHeightReached)); // Should receive nothing from PohRecorder b/c record failed assert!(entry_receiver.try_recv().is_err()); @@ -1389,6 +1446,8 @@ mod tests { bank_forks, &Arc::new(PrioritizationFeeCache::new(0u64)), false, + HashSet::default(), + BundleAccountLocker::default(), ); let keypairs = (0..100).map(|_| Keypair::new()).collect_vec(); diff --git a/core/src/banking_stage/committer.rs b/core/src/banking_stage/committer.rs index 0ca1304a45..63f6921024 100644 --- a/core/src/banking_stage/committer.rs +++ b/core/src/banking_stage/committer.rs @@ -12,15 +12,13 @@ use { transaction_batch::TransactionBatch, vote_sender_types::ReplayVoteSender, }, - solana_sdk::{hash::Hash, pubkey::Pubkey, saturating_add_assign}, + solana_sdk::{hash::Hash, saturating_add_assign}, solana_svm::{ account_loader::TransactionLoadResult, transaction_results::{TransactionExecutionResult, TransactionResults}, }, - solana_transaction_status::{ - token_balances::TransactionTokenBalancesSet, TransactionTokenBalance, - }, - std::{collections::HashMap, sync::Arc}, + solana_transaction_status::{token_balances::TransactionTokenBalancesSet, PreBalanceInfo}, + std::sync::Arc, }; #[derive(Clone, Debug, PartialEq, Eq)] @@ -32,13 +30,6 @@ pub enum CommitTransactionDetails { NotCommitted, } -#[derive(Default)] -pub(super) struct PreBalanceInfo { - pub native: Vec>, - pub token: Vec>, - pub mint_decimals: HashMap, -} - #[derive(Clone)] pub struct Committer { transaction_status_sender: Option, @@ -156,7 +147,7 @@ impl Committer { let txs = batch.sanitized_transactions().to_vec(); let post_balances = bank.collect_balances(batch); let post_token_balances = - collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals); + collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals, None); let mut transaction_index = starting_transaction_index.unwrap_or_default(); let batch_transaction_indexes: Vec<_> = tx_results .execution_results diff --git a/core/src/banking_stage/consume_worker.rs b/core/src/banking_stage/consume_worker.rs index 57a4778d32..ad91be1cd8 100644 --- a/core/src/banking_stage/consume_worker.rs +++ b/core/src/banking_stage/consume_worker.rs @@ -697,11 +697,14 @@ impl ConsumeWorkerTransactionErrorMetrics { mod tests { use { super::*, - crate::banking_stage::{ - committer::Committer, - qos_service::QosService, - scheduler_messages::{TransactionBatchId, TransactionId}, - tests::{create_slow_genesis_config, sanitize_transactions, simulate_poh}, + crate::{ + banking_stage::{ + committer::Committer, + qos_service::QosService, + scheduler_messages::{TransactionBatchId, TransactionId}, + tests::{create_slow_genesis_config, sanitize_transactions, simulate_poh}, + }, + bundle_stage::bundle_account_locker::BundleAccountLocker, }, crossbeam_channel::unbounded, solana_ledger::{ @@ -718,6 +721,7 @@ mod tests { signature::Keypair, system_transaction, }, std::{ + collections::HashSet, sync::{atomic::AtomicBool, RwLock}, thread::JoinHandle, }, @@ -773,7 +777,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let (consume_sender, consume_receiver) = unbounded(); let (consumed_sender, consumed_receiver) = unbounded(); diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 9a0108a385..3a9e602e87 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -1,6 +1,6 @@ use { super::{ - committer::{CommitTransactionDetails, Committer, PreBalanceInfo}, + committer::{CommitTransactionDetails, Committer}, immutable_deserialized_packet::ImmutableDeserializedPacket, leader_slot_metrics::{LeaderSlotMetricsTracker, ProcessTransactionsSummary}, leader_slot_timing_metrics::LeaderExecuteAndCommitTimings, @@ -8,6 +8,7 @@ use { unprocessed_transaction_storage::{ConsumeScannerPayload, UnprocessedTransactionStorage}, BankingStageStats, }, + crate::bundle_stage::bundle_account_locker::BundleAccountLocker, itertools::Itertools, solana_compute_budget::compute_budget_processor::process_compute_budget_instructions, solana_ledger::token_balances::collect_token_balances, @@ -25,6 +26,7 @@ use { clock::{Slot, FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, MAX_PROCESSING_AGE}, feature_set, message::SanitizedMessage, + pubkey::Pubkey, saturating_add_assign, timing::timestamp, transaction::{self, AddressLoader, SanitizedTransaction, TransactionError}, @@ -34,8 +36,9 @@ use { transaction_error_metrics::TransactionErrorMetrics, transaction_processor::{ExecutionRecordingConfig, TransactionProcessingConfig}, }, - solana_timings::ExecuteTimings, + solana_transaction_status::PreBalanceInfo, std::{ + collections::HashSet, sync::{atomic::Ordering, Arc}, time::Instant, }, @@ -78,6 +81,8 @@ pub struct Consumer { transaction_recorder: TransactionRecorder, qos_service: QosService, log_messages_bytes_limit: Option, + blacklisted_accounts: HashSet, + bundle_account_locker: BundleAccountLocker, } impl Consumer { @@ -86,12 +91,16 @@ impl Consumer { transaction_recorder: TransactionRecorder, qos_service: QosService, log_messages_bytes_limit: Option, + blacklisted_accounts: HashSet, + bundle_account_locker: BundleAccountLocker, ) -> Self { Self { committer, transaction_recorder, qos_service, log_messages_bytes_limit, + blacklisted_accounts, + bundle_account_locker, } } @@ -121,6 +130,7 @@ impl Consumer { packets_to_process, ) }, + &self.blacklisted_accounts, ); if reached_end_of_slot { @@ -485,20 +495,26 @@ impl Consumer { cost_model_us, ) = measure_us!(self.qos_service.select_and_accumulate_transaction_costs( bank, + &mut bank.write_cost_tracker().unwrap(), txs, pre_results )); // Only lock accounts for those transactions are selected for the block; // Once accounts are locked, other threads cannot encode transactions that will modify the - // same account state + // same account state. + // BundleAccountLocker is used to prevent race conditions with bundled transactions from bundle stage + let bundle_account_locks = self.bundle_account_locker.account_locks(); let (batch, lock_us) = measure_us!(bank.prepare_sanitized_batch_with_results( txs, transaction_qos_cost_results.iter().map(|r| match r { Ok(_cost) => Ok(()), Err(err) => Err(err.clone()), - }) + }), + Some(&bundle_account_locks.read_locks()), + Some(&bundle_account_locks.write_locks()) )); + drop(bundle_account_locks); // retryable_txs includes AccountInUse, WouldExceedMaxBlockCostLimit // WouldExceedMaxAccountCostLimit, WouldExceedMaxVoteCostLimit @@ -530,8 +546,9 @@ impl Consumer { .iter_mut() .for_each(|x| *x += chunk_offset); - let (cu, us) = - Self::accumulate_execute_units_and_time(&execute_and_commit_timings.execute_timings); + let (cu, us) = execute_and_commit_timings + .execute_timings + .accumulate_execute_units_and_time(); self.qos_service.accumulate_actual_execute_cu(cu); self.qos_service.accumulate_actual_execute_time(us); @@ -568,7 +585,7 @@ impl Consumer { if transaction_status_sender_enabled { pre_balance_info.native = bank.collect_balances(batch); pre_balance_info.token = - collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals) + collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals, None) } }); execute_and_commit_timings.collect_balances_us = collect_balances_us; @@ -647,7 +664,7 @@ impl Consumer { let (record_transactions_summary, record_us) = measure_us!(self .transaction_recorder - .record_transactions(bank.slot(), executed_transactions)); + .record_transactions(bank.slot(), vec![executed_transactions])); execute_and_commit_timings.record_us = record_us; let RecordTransactionsSummary { @@ -770,20 +787,6 @@ impl Consumer { ) } - fn accumulate_execute_units_and_time(execute_timings: &ExecuteTimings) -> (u64, u64) { - execute_timings.details.per_program_timings.values().fold( - (0, 0), - |(units, times), program_timings| { - ( - units - .saturating_add(program_timings.accumulated_units) - .saturating_add(program_timings.total_errored_units), - times.saturating_add(program_timings.accumulated_us), - ) - }, - ) - } - /// This function filters pending packets that are still valid /// # Arguments /// * `transactions` - a batch of transactions deserialized from packets @@ -877,7 +880,7 @@ mod tests { transaction::{MessageHash, Transaction, VersionedTransaction}, }, solana_svm::account_loader::CheckedTransactionDetails, - solana_timings::ProgramTiming, + solana_timings::{ExecuteTimings, ProgramTiming}, solana_transaction_status::{TransactionStatusMeta, VersionedTransactionWithStatusMeta}, std::{ borrow::Cow, @@ -926,7 +929,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let process_transactions_summary = consumer.process_transactions(&bank, &Instant::now(), &transactions); @@ -1101,7 +1111,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let process_transactions_batch_output = consumer.process_and_record_transactions(&bank, &transactions, 0); @@ -1126,7 +1143,13 @@ mod tests { let mut done = false; // read entries until I find mine, might be ticks... - while let Ok((_bank, (entry, _tick_height))) = entry_receiver.recv() { + while let Ok(WorkingBankEntry { + bank, + entries_ticks, + }) = entry_receiver.recv() + { + assert!(entries_ticks.len() == 1); + let entry = &entries_ticks.first().unwrap().0; if !entry.is_tick() { trace!("got entry"); assert_eq!(entry.transactions.len(), transactions.len()); @@ -1244,11 +1267,10 @@ mod tests { let timeout = Duration::from_millis(10); let record = record_receiver.recv_timeout(timeout); if let Ok(record) = record { - let record_response = poh_recorder.write().unwrap().record( - record.slot, - record.mixin, - record.transactions, - ); + let record_response = poh_recorder + .write() + .unwrap() + .record(record.slot, &record.mixins_txs); poh_recorder.write().unwrap().tick(); if record.sender.send(record_response).is_err() { panic!("Error returning mixin hash"); @@ -1285,7 +1307,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let process_transactions_batch_output = consumer.process_and_record_transactions(&bank, &transactions, 0); @@ -1310,9 +1339,13 @@ mod tests { let mut done = false; // read entries until I find mine, might be ticks... - while let Ok((_bank, (entry, _tick_height))) = entry_receiver.recv() { - if !entry.is_tick() { - assert_eq!(entry.transactions.len(), transactions.len()); + while let Ok(WorkingBankEntry { + bank: _, + entries_ticks, + }) = entry_receiver.recv() + { + if !entries_ticks[0].0.is_tick() { + assert_eq!(entries_ticks[0].0.transactions.len(), transactions.len()); done = true; break; } @@ -1386,7 +1419,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let process_transactions_batch_output = consumer.process_and_record_transactions(&bank, &transactions, 0); @@ -1462,7 +1502,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let get_block_cost = || bank.read_cost_tracker().unwrap().block_cost(); let get_tx_count = || bank.read_cost_tracker().unwrap().transaction_count(); @@ -1625,7 +1672,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let process_transactions_batch_output = consumer.process_and_record_transactions(&bank, &transactions, 0); @@ -1821,7 +1875,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder.clone(), QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder.clone(), + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let process_transactions_summary = consumer.process_transactions(&bank, &Instant::now(), &transactions); @@ -1948,7 +2009,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let _ = consumer.process_and_record_transactions(&bank, &transactions, 0); @@ -2093,7 +2161,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); let _ = consumer.process_and_record_transactions(&bank, &[sanitized_tx.clone()], 0); @@ -2153,7 +2228,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); // When the working bank in poh_recorder is None, no packets should be processed (consume will not be called) assert!(!poh_recorder.read().unwrap().has_bank()); @@ -2238,7 +2320,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); // When the working bank in poh_recorder is None, no packets should be processed assert!(!poh_recorder.read().unwrap().has_bank()); @@ -2290,7 +2379,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); // When the working bank in poh_recorder is None, no packets should be processed (consume will not be called) assert!(!poh_recorder.read().unwrap().has_bank()); @@ -2422,7 +2518,14 @@ mod tests { replay_vote_sender, Arc::new(PrioritizationFeeCache::new(0u64)), ); - let consumer = Consumer::new(committer, recorder, QosService::new(1), None); + let consumer = Consumer::new( + committer, + recorder, + QosService::new(1), + None, + HashSet::default(), + BundleAccountLocker::default(), + ); // When the working bank in poh_recorder is None, no packets should be processed (consume will not be called) assert!(!poh_recorder.read().unwrap().has_bank()); @@ -2491,7 +2594,7 @@ mod tests { expected_units += n * 1000; } - let (units, us) = Consumer::accumulate_execute_units_and_time(&execute_timings); + let (units, us) = execute_timings.accumulate_execute_units_and_time(); assert_eq!(expected_units, units); assert_eq!(expected_us, us); diff --git a/core/src/banking_stage/latest_unprocessed_votes.rs b/core/src/banking_stage/latest_unprocessed_votes.rs index 084e4125b8..ed0f1e50ba 100644 --- a/core/src/banking_stage/latest_unprocessed_votes.rs +++ b/core/src/banking_stage/latest_unprocessed_votes.rs @@ -139,7 +139,7 @@ pub(crate) fn weighted_random_order_by_stake<'a>( } #[derive(Default, Debug)] -pub(crate) struct VoteBatchInsertionMetrics { +pub struct VoteBatchInsertionMetrics { pub(crate) num_dropped_gossip: usize, pub(crate) num_dropped_tpu: usize, } diff --git a/core/src/banking_stage/qos_service.rs b/core/src/banking_stage/qos_service.rs index bf8b7df963..ecc1d8c4b6 100644 --- a/core/src/banking_stage/qos_service.rs +++ b/core/src/banking_stage/qos_service.rs @@ -6,7 +6,9 @@ use { super::{committer::CommitTransactionDetails, BatchedTransactionDetails}, solana_cost_model::{ - cost_model::CostModel, cost_tracker::UpdatedCosts, transaction_cost::TransactionCost, + cost_model::CostModel, + cost_tracker::{CostTracker, UpdatedCosts}, + transaction_cost::TransactionCost, }, solana_measure::measure::Measure, solana_runtime::bank::Bank, @@ -42,6 +44,7 @@ impl QosService { pub fn select_and_accumulate_transaction_costs( &self, bank: &Bank, + cost_tracker: &mut CostTracker, // caller should pass in &mut bank.write_cost_tracker().unwrap() transactions: &[SanitizedTransaction], pre_results: impl Iterator>, ) -> (Vec>, usize) { @@ -50,7 +53,8 @@ impl QosService { let (transactions_qos_cost_results, num_included) = self.select_transactions_per_cost( transactions.iter(), transaction_costs.into_iter(), - bank, + bank.slot(), + cost_tracker, ); self.accumulate_estimated_transaction_costs(&Self::accumulate_batched_transaction_costs( transactions_qos_cost_results.iter(), @@ -96,10 +100,10 @@ impl QosService { &self, transactions: impl Iterator, transactions_costs: impl Iterator>, - bank: &Bank, + slot: Slot, + cost_tracker: &mut CostTracker, ) -> (Vec>, usize) { let mut cost_tracking_time = Measure::start("cost_tracking_time"); - let mut cost_tracker = bank.write_cost_tracker().unwrap(); let mut num_included = 0; let select_results = transactions.zip(transactions_costs) .map(|(tx, cost)| { @@ -107,13 +111,13 @@ impl QosService { Ok(cost) => { match cost_tracker.try_add(&cost) { Ok(UpdatedCosts{updated_block_cost, updated_costliest_account_cost}) => { - debug!("slot {:?}, transaction {:?}, cost {:?}, fit into current block, current block cost {}, updated costliest account cost {}", bank.slot(), tx, cost, updated_block_cost, updated_costliest_account_cost); + debug!("slot {:?}, transaction {:?}, cost {:?}, fit into current block, current block cost {}, updated costliest account cost {}", slot, tx, cost, updated_block_cost, updated_costliest_account_cost); self.metrics.stats.selected_txs_count.fetch_add(1, Ordering::Relaxed); num_included += 1; Ok(cost) }, Err(e) => { - debug!("slot {:?}, transaction {:?}, cost {:?}, not fit into current block, '{:?}'", bank.slot(), tx, cost, e); + debug!("slot {:?}, transaction {:?}, cost {:?}, not fit into current block, '{:?}'", slot, tx, cost, e); Err(TransactionError::from(e)) } } @@ -685,8 +689,12 @@ mod tests { bank.write_cost_tracker() .unwrap() .set_limits(cost_limit, cost_limit, cost_limit); - let (results, num_selected) = - qos_service.select_transactions_per_cost(txs.iter(), txs_costs.into_iter(), &bank); + let (results, num_selected) = qos_service.select_transactions_per_cost( + txs.iter(), + txs_costs.into_iter(), + bank.slot(), + &mut bank.write_cost_tracker().unwrap(), + ); assert_eq!(num_selected, 2); // verify that first transfer tx and first vote are allowed @@ -739,8 +747,12 @@ mod tests { .iter() .map(|cost| cost.as_ref().unwrap().sum()) .sum(); - let (qos_cost_results, _num_included) = - qos_service.select_transactions_per_cost(txs.iter(), txs_costs.into_iter(), &bank); + let (qos_cost_results, _num_included) = qos_service.select_transactions_per_cost( + txs.iter(), + txs_costs.into_iter(), + bank.slot(), + &mut bank.write_cost_tracker().unwrap(), + ); assert_eq!( total_txs_cost, bank.read_cost_tracker().unwrap().block_cost() @@ -804,8 +816,12 @@ mod tests { .iter() .map(|cost| cost.as_ref().unwrap().sum()) .sum(); - let (qos_cost_results, _num_included) = - qos_service.select_transactions_per_cost(txs.iter(), txs_costs.into_iter(), &bank); + let (qos_cost_results, _num_included) = qos_service.select_transactions_per_cost( + txs.iter(), + txs_costs.into_iter(), + bank.slot(), + &mut bank.write_cost_tracker().unwrap(), + ); assert_eq!( total_txs_cost, bank.read_cost_tracker().unwrap().block_cost() @@ -859,8 +875,12 @@ mod tests { .iter() .map(|cost| cost.as_ref().unwrap().sum()) .sum(); - let (qos_cost_results, _num_included) = - qos_service.select_transactions_per_cost(txs.iter(), txs_costs.into_iter(), &bank); + let (qos_cost_results, _num_included) = qos_service.select_transactions_per_cost( + txs.iter(), + txs_costs.into_iter(), + bank.slot(), + &mut bank.write_cost_tracker().unwrap(), + ); assert_eq!( total_txs_cost, bank.read_cost_tracker().unwrap().block_cost() diff --git a/core/src/banking_stage/unprocessed_transaction_storage.rs b/core/src/banking_stage/unprocessed_transaction_storage.rs index e790993dfc..ae487952c8 100644 --- a/core/src/banking_stage/unprocessed_transaction_storage.rs +++ b/core/src/banking_stage/unprocessed_transaction_storage.rs @@ -15,18 +15,29 @@ use { }, BankingStageStats, FilterForwardingResults, ForwardOption, }, + crate::{ + bundle_stage::bundle_stage_leader_metrics::BundleStageLeaderMetrics, + immutable_deserialized_bundle::ImmutableDeserializedBundle, + }, itertools::Itertools, min_max_heap::MinMaxHeap, + solana_bundle::BundleExecutionError, solana_measure::{measure, measure_us}, solana_runtime::bank::Bank, solana_sdk::{ - clock::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, feature_set::FeatureSet, hash::Hash, - saturating_add_assign, transaction::SanitizedTransaction, + bundle::SanitizedBundle, + clock::{Slot, FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET}, + feature_set::FeatureSet, + hash::Hash, + pubkey::Pubkey, + saturating_add_assign, + transaction::SanitizedTransaction, }, solana_svm::transaction_error_metrics::TransactionErrorMetrics, std::{ - collections::HashMap, + collections::{HashMap, HashSet, VecDeque}, sync::{atomic::Ordering, Arc}, + time::Instant, }, }; @@ -41,6 +52,7 @@ const MAX_NUM_VOTES_RECEIVE: usize = 10_000; pub enum UnprocessedTransactionStorage { VoteStorage(VoteStorage), LocalTransactionStorage(ThreadLocalUnprocessedPackets), + BundleStorage(BundleStorage), } #[derive(Debug)] @@ -59,10 +71,11 @@ pub struct VoteStorage { pub enum ThreadType { Voting(VoteSource), Transactions, + Bundles, } #[derive(Debug)] -pub(crate) enum InsertPacketBatchSummary { +pub enum InsertPacketBatchSummary { VoteBatchInsertionMetrics(VoteBatchInsertionMetrics), PacketBatchInsertionMetrics(PacketBatchInsertionMetrics), } @@ -146,6 +159,7 @@ fn consume_scan_should_process_packet( banking_stage_stats: &BankingStageStats, packet: &ImmutableDeserializedPacket, payload: &mut ConsumeScannerPayload, + blacklisted_accounts: &HashSet, ) -> ProcessingDecision { // If end of the slot, return should process (quick loop after reached end of slot) if payload.reached_end_of_slot { @@ -176,6 +190,10 @@ fn consume_scan_should_process_packet( bank.get_transaction_account_lock_limit(), ) .is_err() + || message + .account_keys() + .iter() + .any(|key| blacklisted_accounts.contains(key)) { payload .message_hash_to_transaction @@ -271,10 +289,25 @@ impl UnprocessedTransactionStorage { }) } + pub fn new_bundle_storage() -> Self { + Self::BundleStorage(BundleStorage { + last_update_slot: Slot::default(), + unprocessed_bundle_storage: VecDeque::with_capacity( + BundleStorage::BUNDLE_STORAGE_CAPACITY, + ), + cost_model_buffered_bundle_storage: VecDeque::with_capacity( + BundleStorage::BUNDLE_STORAGE_CAPACITY, + ), + }) + } + pub fn is_empty(&self) -> bool { match self { Self::VoteStorage(vote_storage) => vote_storage.is_empty(), Self::LocalTransactionStorage(transaction_storage) => transaction_storage.is_empty(), + UnprocessedTransactionStorage::BundleStorage(bundle_storage) => { + bundle_storage.is_empty() + } } } @@ -282,6 +315,10 @@ impl UnprocessedTransactionStorage { match self { Self::VoteStorage(vote_storage) => vote_storage.len(), Self::LocalTransactionStorage(transaction_storage) => transaction_storage.len(), + UnprocessedTransactionStorage::BundleStorage(bundle_storage) => { + bundle_storage.unprocessed_bundles_len() + + bundle_storage.cost_model_buffered_bundles_len() + } } } @@ -291,6 +328,7 @@ impl UnprocessedTransactionStorage { Self::LocalTransactionStorage(transaction_storage) => { transaction_storage.get_min_compute_unit_price() } + UnprocessedTransactionStorage::BundleStorage(_) => None, } } @@ -300,6 +338,7 @@ impl UnprocessedTransactionStorage { Self::LocalTransactionStorage(transaction_storage) => { transaction_storage.get_max_compute_unit_price() } + UnprocessedTransactionStorage::BundleStorage(_) => None, } } @@ -310,6 +349,9 @@ impl UnprocessedTransactionStorage { Self::LocalTransactionStorage(transaction_storage) => { transaction_storage.max_receive_size() } + UnprocessedTransactionStorage::BundleStorage(bundle_storage) => { + bundle_storage.max_receive_size() + } } } @@ -336,6 +378,9 @@ impl UnprocessedTransactionStorage { Self::LocalTransactionStorage(transaction_storage) => { transaction_storage.forward_option() } + UnprocessedTransactionStorage::BundleStorage(bundle_storage) => { + bundle_storage.forward_option() + } } } @@ -343,6 +388,16 @@ impl UnprocessedTransactionStorage { match self { Self::LocalTransactionStorage(transaction_storage) => transaction_storage.clear(), // Since we set everything as forwarded this is the same Self::VoteStorage(vote_storage) => vote_storage.clear_forwarded_packets(), + UnprocessedTransactionStorage::BundleStorage(bundle_storage) => { + let _ = bundle_storage.reset(); + } + } + } + + pub fn bundle_storage(&mut self) -> Option<&mut BundleStorage> { + match self { + UnprocessedTransactionStorage::BundleStorage(bundle_stoge) => Some(bundle_stoge), + _ => None, } } @@ -357,6 +412,11 @@ impl UnprocessedTransactionStorage { Self::LocalTransactionStorage(transaction_storage) => InsertPacketBatchSummary::from( transaction_storage.insert_batch(deserialized_packets), ), + UnprocessedTransactionStorage::BundleStorage(_) => { + panic!( + "bundles must be inserted using UnprocessedTransactionStorage::insert_bundle" + ) + } } } @@ -376,6 +436,9 @@ impl UnprocessedTransactionStorage { bank, forward_packet_batches_by_accounts, ), + UnprocessedTransactionStorage::BundleStorage(_) => { + panic!("bundles are not forwarded between leaders") + } } } @@ -389,6 +452,7 @@ impl UnprocessedTransactionStorage { banking_stage_stats: &BankingStageStats, slot_metrics_tracker: &mut LeaderSlotMetricsTracker, processing_function: F, + blacklisted_accounts: &HashSet, ) -> bool where F: FnMut( @@ -403,15 +467,62 @@ impl UnprocessedTransactionStorage { banking_stage_stats, slot_metrics_tracker, processing_function, + blacklisted_accounts, ), Self::VoteStorage(vote_storage) => vote_storage.process_packets( bank, banking_stage_stats, slot_metrics_tracker, processing_function, + blacklisted_accounts, + ), + UnprocessedTransactionStorage::BundleStorage(_) => panic!( + "UnprocessedTransactionStorage::BundleStorage does not support processing packets" ), } } + + #[must_use] + pub fn process_bundles( + &mut self, + bank: Arc, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + blacklisted_accounts: &HashSet, + processing_function: F, + ) -> bool + where + F: FnMut( + &[(ImmutableDeserializedBundle, SanitizedBundle)], + &mut BundleStageLeaderMetrics, + ) -> Vec>, + { + match self { + UnprocessedTransactionStorage::BundleStorage(bundle_storage) => bundle_storage + .process_bundles( + bank, + bundle_stage_leader_metrics, + blacklisted_accounts, + processing_function, + ), + _ => panic!("class does not support processing bundles"), + } + } + + /// Inserts bundles into storage. Only supported for UnprocessedTransactionStorage::BundleStorage + pub(crate) fn insert_bundles( + &mut self, + deserialized_bundles: Vec, + ) -> InsertPacketBundlesSummary { + match self { + UnprocessedTransactionStorage::BundleStorage(bundle_storage) => { + bundle_storage.insert_unprocessed_bundles(deserialized_bundles, true) + } + UnprocessedTransactionStorage::LocalTransactionStorage(_) + | UnprocessedTransactionStorage::VoteStorage(_) => { + panic!("UnprocessedTransactionStorage::insert_bundles only works for type UnprocessedTransactionStorage::BundleStorage"); + } + } + } } impl VoteStorage { @@ -480,6 +591,7 @@ impl VoteStorage { banking_stage_stats: &BankingStageStats, slot_metrics_tracker: &mut LeaderSlotMetricsTracker, mut processing_function: F, + blacklisted_accounts: &HashSet, ) -> bool where F: FnMut( @@ -493,7 +605,13 @@ impl VoteStorage { let should_process_packet = |packet: &Arc, payload: &mut ConsumeScannerPayload| { - consume_scan_should_process_packet(&bank, banking_stage_stats, packet, payload) + consume_scan_should_process_packet( + &bank, + banking_stage_stats, + packet, + payload, + blacklisted_accounts, + ) }; // Based on the stake distribution present in the supplied bank, drain the unprocessed votes @@ -576,6 +694,7 @@ impl ThreadLocalUnprocessedPackets { ThreadType::Transactions => ForwardOption::ForwardTransaction, ThreadType::Voting(VoteSource::Tpu) => ForwardOption::ForwardTpuVote, ThreadType::Voting(VoteSource::Gossip) => ForwardOption::NotForward, + ThreadType::Bundles => ForwardOption::NotForward, } } @@ -901,6 +1020,7 @@ impl ThreadLocalUnprocessedPackets { banking_stage_stats: &BankingStageStats, slot_metrics_tracker: &mut LeaderSlotMetricsTracker, mut processing_function: F, + blacklisted_accounts: &HashSet, ) -> bool where F: FnMut( @@ -915,7 +1035,13 @@ impl ThreadLocalUnprocessedPackets { let should_process_packet = |packet: &Arc, payload: &mut ConsumeScannerPayload| { - consume_scan_should_process_packet(bank, banking_stage_stats, packet, payload) + consume_scan_should_process_packet( + bank, + banking_stage_stats, + packet, + payload, + blacklisted_accounts, + ) }; let mut scanner = create_consume_multi_iterator( &all_packets_to_process, @@ -992,6 +1118,319 @@ impl ThreadLocalUnprocessedPackets { } } +pub struct InsertPacketBundlesSummary { + pub insert_packets_summary: InsertPacketBatchSummary, + pub num_bundles_inserted: usize, + pub num_packets_inserted: usize, + pub num_bundles_dropped: usize, +} + +/// Bundle storage has two deques: one for unprocessed bundles and another for ones that exceeded +/// the cost model and need to get retried next slot. +#[derive(Debug)] +pub struct BundleStorage { + last_update_slot: Slot, + unprocessed_bundle_storage: VecDeque, + // Storage for bundles that exceeded the cost model for the slot they were last attempted + // execution on + cost_model_buffered_bundle_storage: VecDeque, +} + +impl BundleStorage { + pub const BUNDLE_STORAGE_CAPACITY: usize = 1000; + fn is_empty(&self) -> bool { + self.unprocessed_bundle_storage.is_empty() + } + + pub fn unprocessed_bundles_len(&self) -> usize { + self.unprocessed_bundle_storage.len() + } + + pub fn unprocessed_packets_len(&self) -> usize { + self.unprocessed_bundle_storage + .iter() + .map(|b| b.len()) + .sum::() + } + + pub(crate) fn cost_model_buffered_bundles_len(&self) -> usize { + self.cost_model_buffered_bundle_storage.len() + } + + pub(crate) fn cost_model_buffered_packets_len(&self) -> usize { + self.cost_model_buffered_bundle_storage + .iter() + .map(|b| b.len()) + .sum() + } + + pub(crate) fn max_receive_size(&self) -> usize { + self.unprocessed_bundle_storage.capacity() - self.unprocessed_bundle_storage.len() + } + + fn forward_option(&self) -> ForwardOption { + ForwardOption::NotForward + } + + /// Returns the number of unprocessed bundles + cost model buffered cleared + pub fn reset(&mut self) -> (usize, usize) { + let num_unprocessed_bundles = self.unprocessed_bundle_storage.len(); + let num_cost_model_buffered_bundles = self.cost_model_buffered_bundle_storage.len(); + self.unprocessed_bundle_storage.clear(); + self.cost_model_buffered_bundle_storage.clear(); + (num_unprocessed_bundles, num_cost_model_buffered_bundles) + } + + fn insert_bundles( + deque: &mut VecDeque, + deserialized_bundles: Vec, + push_back: bool, + ) -> InsertPacketBundlesSummary { + // deque should be initialized with size [Self::BUNDLE_STORAGE_CAPACITY] + let deque_free_space = Self::BUNDLE_STORAGE_CAPACITY + .checked_sub(deque.len()) + .unwrap(); + let bundles_to_insert_count = std::cmp::min(deque_free_space, deserialized_bundles.len()); + let num_bundles_dropped = deserialized_bundles + .len() + .checked_sub(bundles_to_insert_count) + .unwrap(); + let num_packets_inserted = deserialized_bundles + .iter() + .take(bundles_to_insert_count) + .map(|b| b.len()) + .sum::(); + let num_packets_dropped = deserialized_bundles + .iter() + .skip(bundles_to_insert_count) + .map(|b| b.len()) + .sum::(); + + let to_insert = deserialized_bundles + .into_iter() + .take(bundles_to_insert_count); + if push_back { + deque.extend(to_insert) + } else { + to_insert.for_each(|b| deque.push_front(b)); + } + + InsertPacketBundlesSummary { + insert_packets_summary: PacketBatchInsertionMetrics { + num_dropped_packets: num_packets_dropped, + num_dropped_tracer_packets: 0, + } + .into(), + num_bundles_inserted: bundles_to_insert_count, + num_packets_inserted, + num_bundles_dropped, + } + } + + fn push_front_unprocessed_bundles( + &mut self, + deserialized_bundles: Vec, + ) -> InsertPacketBundlesSummary { + Self::insert_bundles( + &mut self.unprocessed_bundle_storage, + deserialized_bundles, + false, + ) + } + + fn push_back_cost_model_buffered_bundles( + &mut self, + deserialized_bundles: Vec, + ) -> InsertPacketBundlesSummary { + Self::insert_bundles( + &mut self.cost_model_buffered_bundle_storage, + deserialized_bundles, + true, + ) + } + + fn insert_unprocessed_bundles( + &mut self, + deserialized_bundles: Vec, + push_back: bool, + ) -> InsertPacketBundlesSummary { + Self::insert_bundles( + &mut self.unprocessed_bundle_storage, + deserialized_bundles, + push_back, + ) + } + + /// Drains bundles from the queue, sanitizes them to prepare for execution, executes them by + /// calling `processing_function`, then potentially rebuffer them. + pub fn process_bundles( + &mut self, + bank: Arc, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + blacklisted_accounts: &HashSet, + mut processing_function: F, + ) -> bool + where + F: FnMut( + &[(ImmutableDeserializedBundle, SanitizedBundle)], + &mut BundleStageLeaderMetrics, + ) -> Vec>, + { + let sanitized_bundles = self.drain_and_sanitize_bundles( + bank, + bundle_stage_leader_metrics, + blacklisted_accounts, + ); + + debug!("processing {} bundles", sanitized_bundles.len()); + let bundle_execution_results = + processing_function(&sanitized_bundles, bundle_stage_leader_metrics); + + let mut is_slot_over = false; + + let mut rebuffered_bundles = Vec::new(); + + sanitized_bundles + .into_iter() + .zip(bundle_execution_results) + .for_each( + |((deserialized_bundle, sanitized_bundle), result)| match result { + Ok(_) => { + debug!("bundle={} executed ok", sanitized_bundle.bundle_id); + // yippee + } + Err(BundleExecutionError::PohRecordError(e)) => { + // buffer the bundle to the front of the queue to be attempted next slot + debug!( + "bundle={} poh record error: {e:?}", + sanitized_bundle.bundle_id + ); + rebuffered_bundles.push(deserialized_bundle); + is_slot_over = true; + } + Err(BundleExecutionError::BankProcessingTimeLimitReached) => { + // buffer the bundle to the front of the queue to be attempted next slot + debug!("bundle={} bank processing done", sanitized_bundle.bundle_id); + rebuffered_bundles.push(deserialized_bundle); + is_slot_over = true; + } + Err(BundleExecutionError::TransactionFailure(e)) => { + debug!( + "bundle={} execution error: {:?}", + sanitized_bundle.bundle_id, e + ); + // do nothing + } + Err(BundleExecutionError::ExceedsCostModel) => { + // cost model buffered bundles contain most recent bundles at the front of the queue + debug!("bundle={} exceeds cost model", sanitized_bundle.bundle_id); + self.push_back_cost_model_buffered_bundles(vec![deserialized_bundle]); + } + Err(BundleExecutionError::TipError(e)) => { + debug!("bundle={} tip error: {}", sanitized_bundle.bundle_id, e); + // Tip errors are _typically_ due to misconfiguration (except for poh record error, bank processing error, exceeds cost model) + // in order to prevent buffering too many bundles, we'll just drop the bundle + } + Err(BundleExecutionError::LockError) => { + // lock errors are irrecoverable due to malformed transactions + debug!("bundle={} lock error", sanitized_bundle.bundle_id); + } + }, + ); + + // rebuffered bundles are pushed onto deque in reverse order so the first bundle is at the front + for bundle in rebuffered_bundles.into_iter().rev() { + self.push_front_unprocessed_bundles(vec![bundle]); + } + + is_slot_over + } + + /// Drains the unprocessed_bundle_storage, converting bundle packets into SanitizedBundles + fn drain_and_sanitize_bundles( + &mut self, + bank: Arc, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + blacklisted_accounts: &HashSet, + ) -> Vec<(ImmutableDeserializedBundle, SanitizedBundle)> { + let mut error_metrics = TransactionErrorMetrics::default(); + + let start = Instant::now(); + + let mut sanitized_bundles = Vec::new(); + + // on new slot, drain anything that was buffered from last slot + if bank.slot() != self.last_update_slot { + sanitized_bundles.extend( + self.cost_model_buffered_bundle_storage + .drain(..) + .filter_map(|packet_bundle| { + let r = packet_bundle.build_sanitized_bundle( + &bank, + blacklisted_accounts, + &mut error_metrics, + ); + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_sanitize_transaction_result(&r); + + match r { + Ok(sanitized_bundle) => Some((packet_bundle, sanitized_bundle)), + Err(e) => { + debug!( + "bundle id: {} error sanitizing: {}", + packet_bundle.bundle_id(), + e + ); + None + } + } + }), + ); + + self.last_update_slot = bank.slot(); + } + + sanitized_bundles.extend(self.unprocessed_bundle_storage.drain(..).filter_map( + |packet_bundle| { + let r = packet_bundle.build_sanitized_bundle( + &bank, + blacklisted_accounts, + &mut error_metrics, + ); + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_sanitize_transaction_result(&r); + match r { + Ok(sanitized_bundle) => Some((packet_bundle, sanitized_bundle)), + Err(e) => { + debug!( + "bundle id: {} error sanitizing: {}", + packet_bundle.bundle_id(), + e + ); + None + } + } + }, + )); + + let elapsed = start.elapsed().as_micros(); + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_sanitize_bundle_elapsed_us(elapsed as u64); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_transactions_from_packets_us(elapsed as u64); + + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .accumulate_transaction_errors(&error_metrics); + + sanitized_bundles + } +} + #[cfg(test)] mod tests { use { diff --git a/core/src/banking_trace.rs b/core/src/banking_trace.rs index c2b3c38695..96358b245f 100644 --- a/core/src/banking_trace.rs +++ b/core/src/banking_trace.rs @@ -321,6 +321,7 @@ impl BankingTracer { } } +#[derive(Clone)] pub struct TracedSender { label: ChannelLabel, sender: Sender, diff --git a/core/src/bundle_stage.rs b/core/src/bundle_stage.rs new file mode 100644 index 0000000000..e935529df5 --- /dev/null +++ b/core/src/bundle_stage.rs @@ -0,0 +1,434 @@ +//! The `bundle_stage` processes bundles, which are list of transactions to be executed +//! sequentially and atomically. +use { + crate::{ + banking_stage::{ + decision_maker::{BufferedPacketsDecision, DecisionMaker}, + qos_service::QosService, + unprocessed_transaction_storage::UnprocessedTransactionStorage, + }, + bundle_stage::{ + bundle_account_locker::BundleAccountLocker, bundle_consumer::BundleConsumer, + bundle_packet_receiver::BundleReceiver, + bundle_reserved_space_manager::BundleReservedSpaceManager, + bundle_stage_leader_metrics::BundleStageLeaderMetrics, committer::Committer, + }, + packet_bundle::PacketBundle, + proxy::block_engine_stage::BlockBuilderFeeInfo, + tip_manager::TipManager, + }, + crossbeam_channel::{Receiver, RecvTimeoutError}, + solana_cost_model::block_cost_limits::MAX_BLOCK_UNITS, + solana_gossip::cluster_info::ClusterInfo, + solana_ledger::blockstore_processor::TransactionStatusSender, + solana_measure::measure, + solana_poh::poh_recorder::PohRecorder, + solana_runtime::{ + bank_forks::BankForks, prioritization_fee_cache::PrioritizationFeeCache, + vote_sender_types::ReplayVoteSender, + }, + solana_sdk::timing::AtomicInterval, + std::{ + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, RwLock, + }, + thread::{self, Builder, JoinHandle}, + time::{Duration, Instant}, + }, +}; + +pub mod bundle_account_locker; +mod bundle_consumer; +mod bundle_packet_deserializer; +mod bundle_packet_receiver; +mod bundle_reserved_space_manager; +pub(crate) mod bundle_stage_leader_metrics; +mod committer; + +const MAX_BUNDLE_RETRY_DURATION: Duration = Duration::from_millis(10); +const SLOT_BOUNDARY_CHECK_PERIOD: Duration = Duration::from_millis(10); + +// Stats emitted periodically +#[derive(Default)] +pub struct BundleStageLoopMetrics { + last_report: AtomicInterval, + id: u32, + + // total received + num_bundles_received: AtomicU64, + num_packets_received: AtomicU64, + + // newly buffered + newly_buffered_bundles_count: AtomicU64, + + // currently buffered + current_buffered_bundles_count: AtomicU64, + current_buffered_packets_count: AtomicU64, + + // buffered due to cost model + cost_model_buffered_bundles_count: AtomicU64, + cost_model_buffered_packets_count: AtomicU64, + + // number of bundles dropped during insertion + num_bundles_dropped: AtomicU64, + + // timings + receive_and_buffer_bundles_elapsed_us: AtomicU64, + process_buffered_bundles_elapsed_us: AtomicU64, +} + +impl BundleStageLoopMetrics { + fn new(id: u32) -> Self { + BundleStageLoopMetrics { + id, + ..BundleStageLoopMetrics::default() + } + } + + pub fn increment_num_bundles_received(&mut self, count: u64) { + self.num_bundles_received + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_num_packets_received(&mut self, count: u64) { + self.num_packets_received + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_newly_buffered_bundles_count(&mut self, count: u64) { + self.newly_buffered_bundles_count + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_current_buffered_bundles_count(&mut self, count: u64) { + self.current_buffered_bundles_count + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_current_buffered_packets_count(&mut self, count: u64) { + self.current_buffered_packets_count + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_cost_model_buffered_bundles_count(&mut self, count: u64) { + self.cost_model_buffered_bundles_count + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_cost_model_buffered_packets_count(&mut self, count: u64) { + self.cost_model_buffered_packets_count + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_num_bundles_dropped(&mut self, count: u64) { + self.num_bundles_dropped.fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_receive_and_buffer_bundles_elapsed_us(&mut self, count: u64) { + self.receive_and_buffer_bundles_elapsed_us + .fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_process_buffered_bundles_elapsed_us(&mut self, count: u64) { + self.process_buffered_bundles_elapsed_us + .fetch_add(count, Ordering::Relaxed); + } +} + +impl BundleStageLoopMetrics { + fn maybe_report(&mut self, report_interval_ms: u64) { + if self.last_report.should_update(report_interval_ms) { + datapoint_info!( + "bundle_stage-loop_stats", + ("id", self.id, i64), + ( + "num_bundles_received", + self.num_bundles_received.swap(0, Ordering::Acquire) as i64, + i64 + ), + ( + "num_packets_received", + self.num_packets_received.swap(0, Ordering::Acquire) as i64, + i64 + ), + ( + "newly_buffered_bundles_count", + self.newly_buffered_bundles_count.swap(0, Ordering::Acquire) as i64, + i64 + ), + ( + "current_buffered_bundles_count", + self.current_buffered_bundles_count + .swap(0, Ordering::Acquire) as i64, + i64 + ), + ( + "current_buffered_packets_count", + self.current_buffered_packets_count + .swap(0, Ordering::Acquire) as i64, + i64 + ), + ( + "num_bundles_dropped", + self.num_bundles_dropped.swap(0, Ordering::Acquire) as i64, + i64 + ), + ( + "receive_and_buffer_bundles_elapsed_us", + self.receive_and_buffer_bundles_elapsed_us + .swap(0, Ordering::Acquire) as i64, + i64 + ), + ( + "process_buffered_bundles_elapsed_us", + self.process_buffered_bundles_elapsed_us + .swap(0, Ordering::Acquire) as i64, + i64 + ), + ); + } + } +} + +pub struct BundleStage { + bundle_thread: JoinHandle<()>, +} + +impl BundleStage { + #[allow(clippy::new_ret_no_self)] + #[allow(clippy::too_many_arguments)] + pub fn new( + cluster_info: &Arc, + poh_recorder: &Arc>, + bundle_receiver: Receiver>, + transaction_status_sender: Option, + replay_vote_sender: ReplayVoteSender, + log_messages_bytes_limit: Option, + exit: Arc, + tip_manager: TipManager, + bundle_account_locker: BundleAccountLocker, + block_builder_fee_info: &Arc>, + preallocated_bundle_cost: u64, + bank_forks: Arc>, + prioritization_fee_cache: &Arc, + ) -> Self { + Self::start_bundle_thread( + cluster_info, + poh_recorder, + bundle_receiver, + transaction_status_sender, + replay_vote_sender, + log_messages_bytes_limit, + exit, + tip_manager, + bundle_account_locker, + MAX_BUNDLE_RETRY_DURATION, + block_builder_fee_info, + preallocated_bundle_cost, + bank_forks, + prioritization_fee_cache, + ) + } + + pub fn join(self) -> thread::Result<()> { + self.bundle_thread.join() + } + + #[allow(clippy::too_many_arguments)] + fn start_bundle_thread( + cluster_info: &Arc, + poh_recorder: &Arc>, + bundle_receiver: Receiver>, + transaction_status_sender: Option, + replay_vote_sender: ReplayVoteSender, + log_message_bytes_limit: Option, + exit: Arc, + tip_manager: TipManager, + bundle_account_locker: BundleAccountLocker, + max_bundle_retry_duration: Duration, + block_builder_fee_info: &Arc>, + preallocated_bundle_cost: u64, + bank_forks: Arc>, + prioritization_fee_cache: &Arc, + ) -> Self { + const BUNDLE_STAGE_ID: u32 = 10_000; + let poh_recorder = poh_recorder.clone(); + let cluster_info = cluster_info.clone(); + + let mut bundle_receiver = + BundleReceiver::new(BUNDLE_STAGE_ID, bundle_receiver, bank_forks, Some(5)); + + let committer = Committer::new( + transaction_status_sender, + replay_vote_sender, + prioritization_fee_cache.clone(), + ); + let decision_maker = DecisionMaker::new(cluster_info.id(), poh_recorder.clone()); + + let unprocessed_bundle_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let reserved_ticks = poh_recorder + .read() + .unwrap() + .ticks_per_slot() + .saturating_mul(8) + .saturating_div(10); + + // The first 80% of the block, based on poh ticks, has `preallocated_bundle_cost` less compute units. + // The last 20% has has full compute so blockspace is maximized if BundleStage is idle. + let reserved_space = BundleReservedSpaceManager::new( + MAX_BLOCK_UNITS, + preallocated_bundle_cost, + reserved_ticks, + ); + + let consumer = BundleConsumer::new( + committer, + poh_recorder.read().unwrap().new_recorder(), + QosService::new(BUNDLE_STAGE_ID), + log_message_bytes_limit, + tip_manager, + bundle_account_locker, + block_builder_fee_info.clone(), + max_bundle_retry_duration, + cluster_info, + reserved_space, + ); + + let bundle_thread = Builder::new() + .name("solBundleStgTx".to_string()) + .spawn(move || { + Self::process_loop( + &mut bundle_receiver, + decision_maker, + consumer, + BUNDLE_STAGE_ID, + unprocessed_bundle_storage, + exit, + ); + }) + .unwrap(); + + Self { bundle_thread } + } + + #[allow(clippy::too_many_arguments)] + fn process_loop( + bundle_receiver: &mut BundleReceiver, + decision_maker: DecisionMaker, + mut consumer: BundleConsumer, + id: u32, + mut unprocessed_bundle_storage: UnprocessedTransactionStorage, + exit: Arc, + ) { + let mut last_metrics_update = Instant::now(); + + let mut bundle_stage_metrics = BundleStageLoopMetrics::new(id); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(id); + + while !exit.load(Ordering::Relaxed) { + if !unprocessed_bundle_storage.is_empty() + || last_metrics_update.elapsed() >= SLOT_BOUNDARY_CHECK_PERIOD + { + let (_, process_buffered_packets_time) = measure!( + Self::process_buffered_bundles( + &decision_maker, + &mut consumer, + &mut unprocessed_bundle_storage, + &mut bundle_stage_leader_metrics, + ), + "process_buffered_packets", + ); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_process_buffered_packets_us(process_buffered_packets_time.as_us()); + last_metrics_update = Instant::now(); + } + + match bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_bundle_storage, + &mut bundle_stage_metrics, + &mut bundle_stage_leader_metrics, + ) { + Ok(_) | Err(RecvTimeoutError::Timeout) => (), + Err(RecvTimeoutError::Disconnected) => break, + } + + let bundle_storage = unprocessed_bundle_storage.bundle_storage().unwrap(); + bundle_stage_metrics.increment_current_buffered_bundles_count( + bundle_storage.unprocessed_bundles_len() as u64, + ); + bundle_stage_metrics.increment_current_buffered_packets_count( + bundle_storage.unprocessed_packets_len() as u64, + ); + bundle_stage_metrics.increment_cost_model_buffered_bundles_count( + bundle_storage.cost_model_buffered_bundles_len() as u64, + ); + bundle_stage_metrics.increment_cost_model_buffered_packets_count( + bundle_storage.cost_model_buffered_packets_len() as u64, + ); + bundle_stage_metrics.maybe_report(1_000); + } + } + + #[allow(clippy::too_many_arguments)] + fn process_buffered_bundles( + decision_maker: &DecisionMaker, + consumer: &mut BundleConsumer, + unprocessed_bundle_storage: &mut UnprocessedTransactionStorage, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) { + let (decision, make_decision_time) = + measure!(decision_maker.make_consume_or_forward_decision()); + + let (metrics_action, banking_stage_metrics_action) = bundle_stage_leader_metrics + .check_leader_slot_boundary(decision.bank_start(), Some(unprocessed_bundle_storage)); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_make_decision_us(make_decision_time.as_us()); + + match decision { + // BufferedPacketsDecision::Consume means this leader is scheduled to be running at the moment. + // Execute, record, and commit as many bundles possible given time, compute, and other constraints. + BufferedPacketsDecision::Consume(bank_start) => { + // Take metrics action before consume packets (potentially resetting the + // slot metrics tracker to the next slot) so that we don't count the + // packet processing metrics from the next slot towards the metrics + // of the previous slot + bundle_stage_leader_metrics + .apply_action(metrics_action, banking_stage_metrics_action); + + let (_, consume_buffered_packets_time) = measure!( + consumer.consume_buffered_bundles( + &bank_start, + unprocessed_bundle_storage, + bundle_stage_leader_metrics, + ), + "consume_buffered_bundles", + ); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_consume_buffered_packets_us(consume_buffered_packets_time.as_us()); + } + // BufferedPacketsDecision::Forward means the leader is slot is far away. + // Bundles aren't forwarded because it breaks atomicity guarantees, so just drop them. + BufferedPacketsDecision::Forward => { + let (_num_bundles_cleared, _num_cost_model_buffered_bundles) = + unprocessed_bundle_storage.bundle_storage().unwrap().reset(); + + // TODO (LB): add metrics here for how many bundles were cleared + + bundle_stage_leader_metrics + .apply_action(metrics_action, banking_stage_metrics_action); + } + // BufferedPacketsDecision::ForwardAndHold | BufferedPacketsDecision::Hold means the validator + // is approaching the leader slot, hold bundles. Also, bundles aren't forwarded because it breaks + // atomicity guarantees + BufferedPacketsDecision::ForwardAndHold | BufferedPacketsDecision::Hold => { + bundle_stage_leader_metrics + .apply_action(metrics_action, banking_stage_metrics_action); + } + } + } +} diff --git a/core/src/bundle_stage/bundle_account_locker.rs b/core/src/bundle_stage/bundle_account_locker.rs new file mode 100644 index 0000000000..82070e3d3a --- /dev/null +++ b/core/src/bundle_stage/bundle_account_locker.rs @@ -0,0 +1,326 @@ +//! Handles pre-locking bundle accounts so that accounts bundles touch can be reserved ahead +// of time for execution. Also, ensures that ALL accounts mentioned across a bundle are locked +// to avoid race conditions between BundleStage and BankingStage. +// +// For instance, imagine a bundle with three transactions and the set of accounts for each transaction +// is: {{A, B}, {B, C}, {C, D}}. We need to lock A, B, and C even though only one is executed at a time. +// Imagine BundleStage is in the middle of processing {C, D} and we didn't have a lock on accounts {A, B, C}. +// In this situation, there's a chance that BankingStage can process a transaction containing A or B +// and commit the results before the bundle completes. By the time the bundle commits the new account +// state for {A, B, C}, A and B would be incorrect and the entries containing the bundle would be +// replayed improperly and that leader would have produced an invalid block. +use { + solana_runtime::bank::Bank, + solana_sdk::{bundle::SanitizedBundle, pubkey::Pubkey, transaction::TransactionAccountLocks}, + std::{ + collections::{hash_map::Entry, HashMap, HashSet}, + sync::{Arc, Mutex, MutexGuard}, + }, + thiserror::Error, +}; + +#[derive(Clone, Error, Debug)] +pub enum BundleAccountLockerError { + #[error("locking error")] + LockingError, +} + +pub type BundleAccountLockerResult = Result; + +pub struct LockedBundle<'a, 'b> { + bundle_account_locker: &'a BundleAccountLocker, + sanitized_bundle: &'b SanitizedBundle, + bank: Arc, +} + +impl<'a, 'b> LockedBundle<'a, 'b> { + pub fn new( + bundle_account_locker: &'a BundleAccountLocker, + sanitized_bundle: &'b SanitizedBundle, + bank: &Arc, + ) -> Self { + Self { + bundle_account_locker, + sanitized_bundle, + bank: bank.clone(), + } + } + + pub fn sanitized_bundle(&self) -> &SanitizedBundle { + self.sanitized_bundle + } +} + +// Automatically unlock bundle accounts when destructed +impl<'a, 'b> Drop for LockedBundle<'a, 'b> { + fn drop(&mut self) { + let _ = self + .bundle_account_locker + .unlock_bundle_accounts(self.sanitized_bundle, &self.bank); + } +} + +#[derive(Default, Clone)] +pub struct BundleAccountLocks { + read_locks: HashMap, + write_locks: HashMap, +} + +impl BundleAccountLocks { + pub fn read_locks(&self) -> HashSet { + self.read_locks.keys().cloned().collect() + } + + pub fn write_locks(&self) -> HashSet { + self.write_locks.keys().cloned().collect() + } + + pub fn lock_accounts( + &mut self, + read_locks: HashMap, + write_locks: HashMap, + ) { + for (acc, count) in read_locks { + *self.read_locks.entry(acc).or_insert(0) += count; + } + for (acc, count) in write_locks { + *self.write_locks.entry(acc).or_insert(0) += count; + } + } + + pub fn unlock_accounts( + &mut self, + read_locks: HashMap, + write_locks: HashMap, + ) { + for (acc, count) in read_locks { + if let Entry::Occupied(mut entry) = self.read_locks.entry(acc) { + let val = entry.get_mut(); + *val = val.saturating_sub(count); + if entry.get() == &0 { + let _ = entry.remove(); + } + } else { + warn!("error unlocking read-locked account, account: {:?}", acc); + } + } + for (acc, count) in write_locks { + if let Entry::Occupied(mut entry) = self.write_locks.entry(acc) { + let val = entry.get_mut(); + *val = val.saturating_sub(count); + if entry.get() == &0 { + let _ = entry.remove(); + } + } else { + warn!("error unlocking write-locked account, account: {:?}", acc); + } + } + } +} + +#[derive(Clone, Default)] +pub struct BundleAccountLocker { + account_locks: Arc>, +} + +impl BundleAccountLocker { + /// used in BankingStage during TransactionBatch construction to ensure that BankingStage + /// doesn't lock anything currently locked in the BundleAccountLocker + pub fn read_locks(&self) -> HashSet { + self.account_locks.lock().unwrap().read_locks() + } + + /// used in BankingStage during TransactionBatch construction to ensure that BankingStage + /// doesn't lock anything currently locked in the BundleAccountLocker + pub fn write_locks(&self) -> HashSet { + self.account_locks.lock().unwrap().write_locks() + } + + /// used in BankingStage during TransactionBatch construction to ensure that BankingStage + /// doesn't lock anything currently locked in the BundleAccountLocker + pub fn account_locks(&self) -> MutexGuard { + self.account_locks.lock().unwrap() + } + + /// Prepares a locked bundle and returns a LockedBundle containing locked accounts. + /// When a LockedBundle is dropped, the accounts are automatically unlocked + pub fn prepare_locked_bundle<'a, 'b>( + &'a self, + sanitized_bundle: &'b SanitizedBundle, + bank: &Arc, + ) -> BundleAccountLockerResult> { + let (read_locks, write_locks) = Self::get_read_write_locks(sanitized_bundle, bank)?; + + self.account_locks + .lock() + .unwrap() + .lock_accounts(read_locks, write_locks); + Ok(LockedBundle::new(self, sanitized_bundle, bank)) + } + + /// Unlocks bundle accounts. Note that LockedBundle::drop will auto-drop the bundle account locks + fn unlock_bundle_accounts( + &self, + sanitized_bundle: &SanitizedBundle, + bank: &Bank, + ) -> BundleAccountLockerResult<()> { + let (read_locks, write_locks) = Self::get_read_write_locks(sanitized_bundle, bank)?; + + self.account_locks + .lock() + .unwrap() + .unlock_accounts(read_locks, write_locks); + Ok(()) + } + + /// Returns the read and write locks for this bundle + /// Each lock type contains a HashMap which maps Pubkey to number of locks held + fn get_read_write_locks( + bundle: &SanitizedBundle, + bank: &Bank, + ) -> BundleAccountLockerResult<(HashMap, HashMap)> { + let transaction_locks: Vec = bundle + .transactions + .iter() + .filter_map(|tx| { + tx.get_account_locks(bank.get_transaction_account_lock_limit()) + .ok() + }) + .collect(); + + if transaction_locks.len() != bundle.transactions.len() { + return Err(BundleAccountLockerError::LockingError); + } + + let bundle_read_locks = transaction_locks + .iter() + .flat_map(|tx| tx.readonly.iter().map(|a| **a)); + let bundle_read_locks = + bundle_read_locks + .into_iter() + .fold(HashMap::new(), |mut map, acc| { + *map.entry(acc).or_insert(0) += 1; + map + }); + + let bundle_write_locks = transaction_locks + .iter() + .flat_map(|tx| tx.writable.iter().map(|a| **a)); + let bundle_write_locks = + bundle_write_locks + .into_iter() + .fold(HashMap::new(), |mut map, acc| { + *map.entry(acc).or_insert(0) += 1; + map + }); + + Ok((bundle_read_locks, bundle_write_locks)) + } +} + +#[cfg(test)] +mod tests { + use { + crate::{ + bundle_stage::bundle_account_locker::BundleAccountLocker, + immutable_deserialized_bundle::ImmutableDeserializedBundle, + packet_bundle::PacketBundle, + }, + solana_ledger::genesis_utils::create_genesis_config, + solana_perf::packet::PacketBatch, + solana_runtime::{bank::Bank, genesis_utils::GenesisConfigInfo}, + solana_sdk::{ + packet::Packet, signature::Signer, signer::keypair::Keypair, system_program, + system_transaction::transfer, transaction::VersionedTransaction, + }, + solana_svm::transaction_error_metrics::TransactionErrorMetrics, + std::collections::HashSet, + }; + + #[test] + fn test_simple_lock_bundles() { + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(2); + let (bank, _) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let bundle_account_locker = BundleAccountLocker::default(); + + let kp0 = Keypair::new(); + let kp1 = Keypair::new(); + + let tx0 = VersionedTransaction::from(transfer( + &mint_keypair, + &kp0.pubkey(), + 1, + genesis_config.hash(), + )); + let tx1 = VersionedTransaction::from(transfer( + &mint_keypair, + &kp1.pubkey(), + 1, + genesis_config.hash(), + )); + + let mut packet_bundle0 = PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data(None, &tx0).unwrap()]), + bundle_id: tx0.signatures[0].to_string(), + }; + let mut packet_bundle1 = PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data(None, &tx1).unwrap()]), + bundle_id: tx1.signatures[0].to_string(), + }; + + let mut transaction_errors = TransactionErrorMetrics::default(); + + let sanitized_bundle0 = ImmutableDeserializedBundle::new(&mut packet_bundle0, None) + .unwrap() + .build_sanitized_bundle(&bank, &HashSet::default(), &mut transaction_errors) + .expect("sanitize bundle 0"); + let sanitized_bundle1 = ImmutableDeserializedBundle::new(&mut packet_bundle1, None) + .unwrap() + .build_sanitized_bundle(&bank, &HashSet::default(), &mut transaction_errors) + .expect("sanitize bundle 1"); + + let locked_bundle0 = bundle_account_locker + .prepare_locked_bundle(&sanitized_bundle0, &bank) + .unwrap(); + + assert_eq!( + bundle_account_locker.write_locks(), + HashSet::from_iter([mint_keypair.pubkey(), kp0.pubkey()]) + ); + assert_eq!( + bundle_account_locker.read_locks(), + HashSet::from_iter([system_program::id()]) + ); + + let locked_bundle1 = bundle_account_locker + .prepare_locked_bundle(&sanitized_bundle1, &bank) + .unwrap(); + assert_eq!( + bundle_account_locker.write_locks(), + HashSet::from_iter([mint_keypair.pubkey(), kp0.pubkey(), kp1.pubkey()]) + ); + assert_eq!( + bundle_account_locker.read_locks(), + HashSet::from_iter([system_program::id()]) + ); + + drop(locked_bundle0); + assert_eq!( + bundle_account_locker.write_locks(), + HashSet::from_iter([mint_keypair.pubkey(), kp1.pubkey()]) + ); + assert_eq!( + bundle_account_locker.read_locks(), + HashSet::from_iter([system_program::id()]) + ); + + drop(locked_bundle1); + assert!(bundle_account_locker.write_locks().is_empty()); + assert!(bundle_account_locker.read_locks().is_empty()); + } +} diff --git a/core/src/bundle_stage/bundle_consumer.rs b/core/src/bundle_stage/bundle_consumer.rs new file mode 100644 index 0000000000..b8d71edaf6 --- /dev/null +++ b/core/src/bundle_stage/bundle_consumer.rs @@ -0,0 +1,1584 @@ +use { + crate::{ + banking_stage::{ + committer::CommitTransactionDetails, leader_slot_metrics::ProcessTransactionsSummary, + leader_slot_timing_metrics::LeaderExecuteAndCommitTimings, qos_service::QosService, + unprocessed_transaction_storage::UnprocessedTransactionStorage, + }, + bundle_stage::{ + bundle_account_locker::{BundleAccountLocker, LockedBundle}, + bundle_reserved_space_manager::BundleReservedSpaceManager, + bundle_stage_leader_metrics::BundleStageLeaderMetrics, + committer::Committer, + }, + consensus_cache_updater::ConsensusCacheUpdater, + immutable_deserialized_bundle::ImmutableDeserializedBundle, + proxy::block_engine_stage::BlockBuilderFeeInfo, + tip_manager::TipManager, + }, + solana_bundle::{ + bundle_execution::{load_and_execute_bundle, BundleExecutionMetrics}, + BundleExecutionError, BundleExecutionResult, TipError, + }, + solana_cost_model::transaction_cost::TransactionCost, + solana_gossip::cluster_info::ClusterInfo, + solana_measure::{measure, measure_us}, + solana_poh::poh_recorder::{BankStart, RecordTransactionsSummary, TransactionRecorder}, + solana_runtime::bank::Bank, + solana_sdk::{ + bundle::SanitizedBundle, + clock::{Slot, MAX_PROCESSING_AGE}, + pubkey::Pubkey, + transaction::{self}, + }, + solana_svm::transaction_error_metrics::TransactionErrorMetrics, + std::{ + collections::HashSet, + sync::{Arc, Mutex}, + time::{Duration, Instant}, + }, +}; + +pub struct ExecuteRecordCommitResult { + commit_transaction_details: Vec, + result: BundleExecutionResult<()>, + execution_metrics: BundleExecutionMetrics, + execute_and_commit_timings: LeaderExecuteAndCommitTimings, + transaction_error_counter: TransactionErrorMetrics, +} + +pub struct BundleConsumer { + committer: Committer, + transaction_recorder: TransactionRecorder, + qos_service: QosService, + log_messages_bytes_limit: Option, + + consensus_cache_updater: ConsensusCacheUpdater, + + tip_manager: TipManager, + last_tip_update_slot: Slot, + + blacklisted_accounts: HashSet, + + // Manages account locks across multiple transactions within a bundle to prevent race conditions + // with BankingStage + bundle_account_locker: BundleAccountLocker, + + block_builder_fee_info: Arc>, + + max_bundle_retry_duration: Duration, + + cluster_info: Arc, + + reserved_space: BundleReservedSpaceManager, +} + +impl BundleConsumer { + #[allow(clippy::too_many_arguments)] + pub fn new( + committer: Committer, + transaction_recorder: TransactionRecorder, + qos_service: QosService, + log_messages_bytes_limit: Option, + tip_manager: TipManager, + bundle_account_locker: BundleAccountLocker, + block_builder_fee_info: Arc>, + max_bundle_retry_duration: Duration, + cluster_info: Arc, + reserved_space: BundleReservedSpaceManager, + ) -> Self { + Self { + committer, + transaction_recorder, + qos_service, + log_messages_bytes_limit, + consensus_cache_updater: ConsensusCacheUpdater::default(), + tip_manager, + // MAX because sending tips during slot 0 in tests doesn't work + last_tip_update_slot: u64::MAX, + blacklisted_accounts: HashSet::default(), + bundle_account_locker, + block_builder_fee_info, + max_bundle_retry_duration, + cluster_info, + reserved_space, + } + } + + // A bundle is a series of transactions to be executed sequentially, atomically, and all-or-nothing. + // Sequentially: + // - Transactions are executed in order + // Atomically: + // - All transactions in a bundle get recoded to PoH and committed to the bank in the same slot. Account locks + // for all accounts in all transactions in a bundle are held during the entire execution to remove POH record race conditions + // with transactions in BankingStage. + // All-or-nothing: + // - All transactions are committed or none. Modified state for the entire bundle isn't recorded to PoH and committed to the + // bank until all transactions in the bundle have executed. + // + // Some corner cases to be aware of when working with BundleStage: + // A bundle is not allowed to call the Tip Payment program in a bundle (or BankingStage). + // - This is to avoid stealing of tips by malicious parties with bundles that crank the tip + // payment program and set the tip receiver to themself. + // A bundle is not allowed to touch consensus-related accounts + // - This is to avoid stalling the voting BankingStage threads. + pub fn consume_buffered_bundles( + &mut self, + bank_start: &BankStart, + unprocessed_transaction_storage: &mut UnprocessedTransactionStorage, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) { + self.maybe_update_blacklist(bank_start); + self.reserved_space.tick(&bank_start.working_bank); + + let reached_end_of_slot = unprocessed_transaction_storage.process_bundles( + bank_start.working_bank.clone(), + bundle_stage_leader_metrics, + &self.blacklisted_accounts, + |bundles, bundle_stage_leader_metrics| { + Self::do_process_bundles( + &self.bundle_account_locker, + &self.tip_manager, + &mut self.last_tip_update_slot, + &self.cluster_info, + &self.block_builder_fee_info, + &self.committer, + &self.transaction_recorder, + &self.qos_service, + &self.log_messages_bytes_limit, + self.max_bundle_retry_duration, + &self.reserved_space, + bundles, + bank_start, + bundle_stage_leader_metrics, + ) + }, + ); + + if reached_end_of_slot { + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .set_end_of_slot_unprocessed_buffer_len( + unprocessed_transaction_storage.len() as u64 + ); + } + } + + /// Blacklist is updated with the tip payment program + any consensus accounts. + fn maybe_update_blacklist(&mut self, bank_start: &BankStart) { + if self + .consensus_cache_updater + .maybe_update(&bank_start.working_bank) + { + self.blacklisted_accounts = self + .consensus_cache_updater + .consensus_accounts_cache() + .union(&HashSet::from_iter([self + .tip_manager + .tip_payment_program_id()])) + .cloned() + .collect(); + + debug!( + "updated blacklist with {} accounts", + self.blacklisted_accounts.len() + ); + } + } + + #[allow(clippy::too_many_arguments)] + fn do_process_bundles( + bundle_account_locker: &BundleAccountLocker, + tip_manager: &TipManager, + last_tip_updated_slot: &mut Slot, + cluster_info: &Arc, + block_builder_fee_info: &Arc>, + committer: &Committer, + recorder: &TransactionRecorder, + qos_service: &QosService, + log_messages_bytes_limit: &Option, + max_bundle_retry_duration: Duration, + reserved_space: &BundleReservedSpaceManager, + bundles: &[(ImmutableDeserializedBundle, SanitizedBundle)], + bank_start: &BankStart, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) -> Vec> { + // BundleAccountLocker holds RW locks for ALL accounts in ALL transactions within a single bundle. + // By pre-locking bundles before they're ready to be processed, it will prevent BankingStage from + // grabbing those locks so BundleStage can process as fast as possible. + // A LockedBundle is similar to TransactionBatch; once its dropped the locks are released. + #[allow(clippy::needless_collect)] + let (locked_bundle_results, locked_bundles_elapsed) = measure!( + bundles + .iter() + .map(|(_, sanitized_bundle)| { + bundle_account_locker + .prepare_locked_bundle(sanitized_bundle, &bank_start.working_bank) + }) + .collect::>(), + "locked_bundles_elapsed" + ); + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_locked_bundle_elapsed_us(locked_bundles_elapsed.as_us()); + + let (execution_results, execute_locked_bundles_elapsed) = measure!(locked_bundle_results + .into_iter() + .map(|r| match r { + Ok(locked_bundle) => { + let (r, measure) = measure_us!(Self::process_bundle( + bundle_account_locker, + tip_manager, + last_tip_updated_slot, + cluster_info, + block_builder_fee_info, + committer, + recorder, + qos_service, + log_messages_bytes_limit, + max_bundle_retry_duration, + reserved_space, + &locked_bundle, + bank_start, + bundle_stage_leader_metrics, + )); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_process_packets_transactions_us(measure); + r + } + Err(_) => { + Err(BundleExecutionError::LockError) + } + }) + .collect::>()); + + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_execute_locked_bundles_elapsed_us(execute_locked_bundles_elapsed.as_us()); + execution_results.iter().for_each(|result| { + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_bundle_execution_result(result); + }); + + execution_results + } + + #[allow(clippy::too_many_arguments)] + fn process_bundle( + bundle_account_locker: &BundleAccountLocker, + tip_manager: &TipManager, + last_tip_updated_slot: &mut Slot, + cluster_info: &Arc, + block_builder_fee_info: &Arc>, + committer: &Committer, + recorder: &TransactionRecorder, + qos_service: &QosService, + log_messages_bytes_limit: &Option, + max_bundle_retry_duration: Duration, + reserved_space: &BundleReservedSpaceManager, + locked_bundle: &LockedBundle, + bank_start: &BankStart, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) -> Result<(), BundleExecutionError> { + if !Bank::should_bank_still_be_processing_txs( + &bank_start.bank_creation_time, + bank_start.working_bank.ns_per_slot, + ) { + return Err(BundleExecutionError::BankProcessingTimeLimitReached); + } + + if bank_start.working_bank.slot() != *last_tip_updated_slot + && Self::bundle_touches_tip_pdas( + locked_bundle.sanitized_bundle(), + &tip_manager.get_tip_accounts(), + ) + { + let start = Instant::now(); + let result = Self::handle_tip_programs( + bundle_account_locker, + tip_manager, + cluster_info, + block_builder_fee_info, + committer, + recorder, + qos_service, + log_messages_bytes_limit, + max_bundle_retry_duration, + reserved_space, + bank_start, + bundle_stage_leader_metrics, + ); + + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_change_tip_receiver_elapsed_us(start.elapsed().as_micros() as u64); + + result?; + + *last_tip_updated_slot = bank_start.working_bank.slot(); + } + + Self::update_qos_and_execute_record_commit_bundle( + committer, + recorder, + qos_service, + log_messages_bytes_limit, + max_bundle_retry_duration, + reserved_space, + locked_bundle.sanitized_bundle(), + bank_start, + bundle_stage_leader_metrics, + )?; + + Ok(()) + } + + /// The validator needs to manage state on two programs related to tips + #[allow(clippy::too_many_arguments)] + fn handle_tip_programs( + bundle_account_locker: &BundleAccountLocker, + tip_manager: &TipManager, + cluster_info: &Arc, + block_builder_fee_info: &Arc>, + committer: &Committer, + recorder: &TransactionRecorder, + qos_service: &QosService, + log_messages_bytes_limit: &Option, + max_bundle_retry_duration: Duration, + reserved_space: &BundleReservedSpaceManager, + bank_start: &BankStart, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) -> Result<(), BundleExecutionError> { + debug!("handle_tip_programs"); + + // This will setup the tip payment and tip distribution program if they haven't been + // initialized yet, which is typically helpful for local validators. On mainnet and testnet, + // this code should never run. + let keypair = cluster_info.keypair().clone(); + let initialize_tip_programs_bundle = + tip_manager.get_initialize_tip_programs_bundle(&bank_start.working_bank, &keypair); + if let Some(bundle) = initialize_tip_programs_bundle { + debug!( + "initializing tip programs with {} transactions, bundle id: {}", + bundle.transactions.len(), + bundle.bundle_id + ); + + let locked_init_tip_programs_bundle = bundle_account_locker + .prepare_locked_bundle(&bundle, &bank_start.working_bank) + .map_err(|_| BundleExecutionError::TipError(TipError::LockError))?; + + Self::update_qos_and_execute_record_commit_bundle( + committer, + recorder, + qos_service, + log_messages_bytes_limit, + max_bundle_retry_duration, + reserved_space, + locked_init_tip_programs_bundle.sanitized_bundle(), + bank_start, + bundle_stage_leader_metrics, + ) + .map_err(|e| { + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_num_init_tip_account_errors(1); + error!( + "bundle: {} error initializing tip programs: {:?}", + locked_init_tip_programs_bundle.sanitized_bundle().bundle_id, + e + ); + BundleExecutionError::TipError(TipError::InitializeProgramsError) + })?; + + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_num_init_tip_account_ok(1); + } + + // There are two frequently run internal cranks inside the jito-solana validator that have to do with managing MEV tips. + // One is initialize the TipDistributionAccount, which is a validator's "tip piggy bank" for an epoch + // The other is ensuring the tip_receiver is configured correctly to ensure tips are routed to the correct + // address. The validator must drain the tip accounts to the previous tip receiver before setting the tip receiver to + // themselves. + + let kp = cluster_info.keypair().clone(); + let tip_crank_bundle = tip_manager.get_tip_programs_crank_bundle( + &bank_start.working_bank, + &kp, + &block_builder_fee_info.lock().unwrap(), + )?; + debug!("tip_crank_bundle is_some: {}", tip_crank_bundle.is_some()); + + if let Some(bundle) = tip_crank_bundle { + info!( + "bundle id: {} cranking tip programs with {} transactions", + bundle.bundle_id, + bundle.transactions.len() + ); + + let locked_tip_crank_bundle = bundle_account_locker + .prepare_locked_bundle(&bundle, &bank_start.working_bank) + .map_err(|_| BundleExecutionError::TipError(TipError::LockError))?; + + Self::update_qos_and_execute_record_commit_bundle( + committer, + recorder, + qos_service, + log_messages_bytes_limit, + max_bundle_retry_duration, + reserved_space, + locked_tip_crank_bundle.sanitized_bundle(), + bank_start, + bundle_stage_leader_metrics, + ) + .map_err(|e| { + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_num_change_tip_receiver_errors(1); + error!( + "bundle: {} error cranking tip programs: {:?}", + locked_tip_crank_bundle.sanitized_bundle().bundle_id, + e + ); + BundleExecutionError::TipError(TipError::CrankTipError) + })?; + + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_num_change_tip_receiver_ok(1); + } + + Ok(()) + } + + /// Reserves space for the entire bundle up-front to ensure the entire bundle can execute. + /// Rolls back the reserved space if there's not enough blockspace for all transactions in the bundle. + fn reserve_bundle_blockspace( + qos_service: &QosService, + reserved_space: &BundleReservedSpaceManager, + sanitized_bundle: &SanitizedBundle, + bank: &Arc, + ) -> BundleExecutionResult<(Vec>, usize)> { + let mut write_cost_tracker = bank.write_cost_tracker().unwrap(); + + // set the block cost limit to the original block cost limit, run the select + accumulate + // then reset back to the expected block cost limit. this allows bundle stage to potentially + // increase block_compute_limits, allocate the space, and reset the block_cost_limits to + // the reserved space without BankingStage racing to allocate this extra reserved space + write_cost_tracker.set_block_cost_limit(reserved_space.block_cost_limit()); + let (transaction_qos_cost_results, cost_model_throttled_transactions_count) = qos_service + .select_and_accumulate_transaction_costs( + bank, + &mut write_cost_tracker, + &sanitized_bundle.transactions, + std::iter::repeat(Ok(())), + ); + write_cost_tracker.set_block_cost_limit(reserved_space.expected_block_cost_limits(bank)); + drop(write_cost_tracker); + + // rollback all transaction costs if it can't fit and + if transaction_qos_cost_results.iter().any(|c| c.is_err()) { + QosService::remove_or_update_costs(transaction_qos_cost_results.iter(), None, bank); + return Err(BundleExecutionError::ExceedsCostModel); + } + + Ok(( + transaction_qos_cost_results, + cost_model_throttled_transactions_count, + )) + } + + fn update_qos_and_execute_record_commit_bundle( + committer: &Committer, + recorder: &TransactionRecorder, + qos_service: &QosService, + log_messages_bytes_limit: &Option, + max_bundle_retry_duration: Duration, + reserved_space: &BundleReservedSpaceManager, + sanitized_bundle: &SanitizedBundle, + bank_start: &BankStart, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) -> BundleExecutionResult<()> { + debug!( + "bundle: {} reserving blockspace for {} transactions", + sanitized_bundle.bundle_id, + sanitized_bundle.transactions.len() + ); + + let ( + (transaction_qos_cost_results, _cost_model_throttled_transactions_count), + cost_model_elapsed_us, + ) = measure_us!(Self::reserve_bundle_blockspace( + qos_service, + reserved_space, + sanitized_bundle, + &bank_start.working_bank + )?); + + debug!( + "bundle: {} executing, recording, and committing", + sanitized_bundle.bundle_id + ); + + let (result, process_transactions_us) = measure_us!(Self::execute_record_commit_bundle( + committer, + recorder, + log_messages_bytes_limit, + max_bundle_retry_duration, + sanitized_bundle, + bank_start, + )); + + bundle_stage_leader_metrics + .bundle_stage_metrics_tracker() + .increment_num_execution_retries(result.execution_metrics.num_retries); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .accumulate_transaction_errors(&result.transaction_error_counter); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_process_transactions_us(process_transactions_us); + + let (cu, us) = result + .execute_and_commit_timings + .execute_timings + .accumulate_execute_units_and_time(); + qos_service.accumulate_actual_execute_cu(cu); + qos_service.accumulate_actual_execute_time(us); + + let num_committed = result + .commit_transaction_details + .iter() + .filter(|c| matches!(c, CommitTransactionDetails::Committed { .. })) + .count(); + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .accumulate_process_transactions_summary(&ProcessTransactionsSummary { + reached_max_poh_height: matches!( + result.result, + Err(BundleExecutionError::BankProcessingTimeLimitReached) + | Err(BundleExecutionError::PohRecordError(_)) + ), + transactions_attempted_execution_count: sanitized_bundle.transactions.len(), + committed_transactions_count: num_committed, + // NOTE: this assumes that bundles are committed all-or-nothing + committed_transactions_with_successful_result_count: num_committed, + failed_commit_count: 0, + retryable_transaction_indexes: vec![], + cost_model_throttled_transactions_count: 0, + cost_model_us: cost_model_elapsed_us, + execute_and_commit_timings: result.execute_and_commit_timings, + error_counters: result.transaction_error_counter, + min_prioritization_fees: 0, // TODO (LB) + max_prioritization_fees: 0, // TODO (LB) + }); + + match result.result { + Ok(_) => { + QosService::remove_or_update_costs( + transaction_qos_cost_results.iter(), + Some(&result.commit_transaction_details), + &bank_start.working_bank, + ); + + qos_service.report_metrics(bank_start.working_bank.slot()); + Ok(()) + } + Err(e) => { + // on bundle failure, none of the transactions are committed, so need to revert + // all compute reserved + QosService::remove_or_update_costs( + transaction_qos_cost_results.iter(), + None, + &bank_start.working_bank, + ); + qos_service.report_metrics(bank_start.working_bank.slot()); + + Err(e) + } + } + } + + fn execute_record_commit_bundle( + committer: &Committer, + recorder: &TransactionRecorder, + log_messages_bytes_limit: &Option, + max_bundle_retry_duration: Duration, + sanitized_bundle: &SanitizedBundle, + bank_start: &BankStart, + ) -> ExecuteRecordCommitResult { + let transaction_status_sender_enabled = committer.transaction_status_sender_enabled(); + + let mut execute_and_commit_timings = LeaderExecuteAndCommitTimings::default(); + + debug!("bundle: {} executing", sanitized_bundle.bundle_id); + let default_accounts = vec![None; sanitized_bundle.transactions.len()]; + let mut bundle_execution_results = load_and_execute_bundle( + &bank_start.working_bank, + sanitized_bundle, + MAX_PROCESSING_AGE, + &max_bundle_retry_duration, + transaction_status_sender_enabled, + log_messages_bytes_limit, + false, + None, + &default_accounts, + &default_accounts, + ); + + let execution_metrics = bundle_execution_results.metrics(); + + execute_and_commit_timings.collect_balances_us = execution_metrics.collect_balances_us; + execute_and_commit_timings.load_execute_us = execution_metrics.load_execute_us; + execute_and_commit_timings + .execute_timings + .accumulate(&execution_metrics.execute_timings); + + let mut transaction_error_counter = TransactionErrorMetrics::default(); + bundle_execution_results + .bundle_transaction_results() + .iter() + .for_each(|r| { + transaction_error_counter + .accumulate(&r.load_and_execute_transactions_output().error_counters); + }); + + debug!( + "bundle: {} executed, is_ok: {}", + sanitized_bundle.bundle_id, + bundle_execution_results.result().is_ok() + ); + + // don't commit bundle if failure executing any part of the bundle + if let Err(e) = bundle_execution_results.result() { + return ExecuteRecordCommitResult { + commit_transaction_details: vec![], + result: Err(e.clone().into()), + execution_metrics, + execute_and_commit_timings, + transaction_error_counter, + }; + } + + let (executed_batches, execution_results_to_transactions_us) = + measure_us!(bundle_execution_results.executed_transaction_batches()); + + debug!( + "bundle: {} recording {} batches of {:?} transactions", + sanitized_bundle.bundle_id, + executed_batches.len(), + executed_batches + .iter() + .map(|b| b.len()) + .collect::>() + ); + + let (freeze_lock, freeze_lock_us) = measure_us!(bank_start.working_bank.freeze_lock()); + execute_and_commit_timings.freeze_lock_us = freeze_lock_us; + + let (last_blockhash, lamports_per_signature) = bank_start + .working_bank + .last_blockhash_and_lamports_per_signature(); + + let ( + RecordTransactionsSummary { + result: record_transactions_result, + record_transactions_timings, + starting_transaction_index, + }, + record_us, + ) = measure_us!( + recorder.record_transactions(bank_start.working_bank.slot(), executed_batches) + ); + + execute_and_commit_timings.record_us = record_us; + execute_and_commit_timings.record_transactions_timings = record_transactions_timings; + execute_and_commit_timings + .record_transactions_timings + .execution_results_to_transactions_us = execution_results_to_transactions_us; + + debug!( + "bundle: {} record result: {}", + sanitized_bundle.bundle_id, + record_transactions_result.is_ok() + ); + + // don't commit bundle if failed to record + if let Err(e) = record_transactions_result { + return ExecuteRecordCommitResult { + commit_transaction_details: vec![], + result: Err(e.into()), + execution_metrics, + execute_and_commit_timings, + transaction_error_counter, + }; + } + + // note: execute_and_commit_timings.commit_us handled inside this function + let (commit_us, commit_bundle_details) = committer.commit_bundle( + &mut bundle_execution_results, + last_blockhash, + lamports_per_signature, + starting_transaction_index, + &bank_start.working_bank, + &mut execute_and_commit_timings, + ); + execute_and_commit_timings.commit_us = commit_us; + + drop(freeze_lock); + + // commit_bundle_details contains transactions that were and were not committed + // given the current implementation only executes, records, and commits bundles + // where all transactions executed, we can filter out the non-committed + // TODO (LB): does this make more sense in commit_bundle for future when failing bundles are accepted? + let commit_transaction_details = commit_bundle_details + .commit_transaction_details + .into_iter() + .flat_map(|commit_details| { + commit_details + .into_iter() + .filter(|d| matches!(d, CommitTransactionDetails::Committed { .. })) + }) + .collect(); + debug!( + "bundle: {} commit details: {:?}", + sanitized_bundle.bundle_id, commit_transaction_details + ); + + ExecuteRecordCommitResult { + commit_transaction_details, + result: Ok(()), + execution_metrics, + execute_and_commit_timings, + transaction_error_counter, + } + } + + /// Returns true if any of the transactions in a bundle mention one of the tip PDAs + fn bundle_touches_tip_pdas(bundle: &SanitizedBundle, tip_pdas: &HashSet) -> bool { + bundle.transactions.iter().any(|tx| { + tx.message() + .account_keys() + .iter() + .any(|a| tip_pdas.contains(a)) + }) + } +} + +#[cfg(test)] +mod tests { + use { + crate::{ + bundle_stage::{ + bundle_account_locker::BundleAccountLocker, bundle_consumer::BundleConsumer, + bundle_packet_deserializer::BundlePacketDeserializer, + bundle_reserved_space_manager::BundleReservedSpaceManager, + bundle_stage_leader_metrics::BundleStageLeaderMetrics, committer::Committer, + QosService, UnprocessedTransactionStorage, + }, + packet_bundle::PacketBundle, + proxy::block_engine_stage::BlockBuilderFeeInfo, + tip_manager::{TipDistributionAccountConfig, TipManager, TipManagerConfig}, + }, + crossbeam_channel::{unbounded, Receiver}, + jito_tip_distribution::sdk::derive_tip_distribution_account_address, + rand::{thread_rng, RngCore}, + solana_cost_model::{block_cost_limits::MAX_BLOCK_UNITS, cost_model::CostModel}, + solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, + solana_ledger::{ + blockstore::Blockstore, genesis_utils::create_genesis_config, + get_tmp_ledger_path_auto_delete, leader_schedule_cache::LeaderScheduleCache, + }, + solana_perf::packet::PacketBatch, + solana_poh::{ + poh_recorder::{PohRecorder, Record, WorkingBankEntry}, + poh_service::PohService, + }, + solana_program_test::programs::spl_programs, + solana_runtime::{ + bank::Bank, + bank_forks::BankForks, + genesis_utils::{create_genesis_config_with_leader_ex, GenesisConfigInfo}, + installed_scheduler_pool::BankWithScheduler, + prioritization_fee_cache::PrioritizationFeeCache, + }, + solana_sdk::{ + bundle::{derive_bundle_id, SanitizedBundle}, + clock::MAX_PROCESSING_AGE, + fee_calculator::{FeeRateGovernor, DEFAULT_TARGET_LAMPORTS_PER_SIGNATURE}, + genesis_config::ClusterType, + hash::Hash, + native_token::sol_to_lamports, + packet::Packet, + poh_config::PohConfig, + pubkey::Pubkey, + rent::Rent, + signature::{Keypair, Signer}, + system_transaction::transfer, + transaction::{SanitizedTransaction, TransactionError, VersionedTransaction}, + vote::state::VoteState, + }, + solana_streamer::socket::SocketAddrSpace, + solana_svm::{ + account_loader::TransactionCheckResult, + transaction_error_metrics::TransactionErrorMetrics, + }, + std::{ + collections::HashSet, + str::FromStr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, RwLock, + }, + thread::{Builder, JoinHandle}, + time::Duration, + }, + }; + + struct TestFixture { + genesis_config_info: GenesisConfigInfo, + leader_keypair: Keypair, + bank: Arc, + exit: Arc, + poh_recorder: Arc>, + poh_simulator: JoinHandle<()>, + entry_receiver: Receiver, + bank_forks: Arc>, + } + + pub(crate) fn simulate_poh( + record_receiver: Receiver, + poh_recorder: &Arc>, + ) -> JoinHandle<()> { + let poh_recorder = poh_recorder.clone(); + let is_exited = poh_recorder.read().unwrap().is_exited.clone(); + let tick_producer = Builder::new() + .name("solana-simulate_poh".to_string()) + .spawn(move || loop { + PohService::read_record_receiver_and_process( + &poh_recorder, + &record_receiver, + Duration::from_millis(10), + ); + if is_exited.load(Ordering::Relaxed) { + break; + } + }); + tick_producer.unwrap() + } + + pub fn create_test_recorder( + bank: &Arc, + blockstore: Arc, + poh_config: Option, + leader_schedule_cache: Option>, + ) -> ( + Arc, + Arc>, + JoinHandle<()>, + Receiver, + ) { + let leader_schedule_cache = match leader_schedule_cache { + Some(provided_cache) => provided_cache, + None => Arc::new(LeaderScheduleCache::new_from_bank(bank)), + }; + let exit = Arc::new(AtomicBool::new(false)); + let poh_config = poh_config.unwrap_or_default(); + + let (mut poh_recorder, entry_receiver, record_receiver) = PohRecorder::new( + bank.tick_height(), + bank.last_blockhash(), + bank.clone(), + Some((4, 4)), + bank.ticks_per_slot(), + blockstore, + &leader_schedule_cache, + &poh_config, + exit.clone(), + ); + poh_recorder.set_bank( + BankWithScheduler::new_without_scheduler(bank.clone()), + false, + ); + + let poh_recorder = Arc::new(RwLock::new(poh_recorder)); + let poh_simulator = simulate_poh(record_receiver, &poh_recorder); + + (exit, poh_recorder, poh_simulator, entry_receiver) + } + + fn create_test_fixture(mint_sol: u64) -> TestFixture { + let mint_keypair = Keypair::new(); + let leader_keypair = Keypair::new(); + let voting_keypair = Keypair::new(); + + let rent = Rent::default(); + + let mut genesis_config = create_genesis_config_with_leader_ex( + sol_to_lamports(mint_sol as f64), + &mint_keypair.pubkey(), + &leader_keypair.pubkey(), + &voting_keypair.pubkey(), + &solana_sdk::pubkey::new_rand(), + rent.minimum_balance(VoteState::size_of()) + sol_to_lamports(1_000_000.0), + sol_to_lamports(1_000_000.0), + FeeRateGovernor { + // Initialize with a non-zero fee + lamports_per_signature: DEFAULT_TARGET_LAMPORTS_PER_SIGNATURE / 2, + ..FeeRateGovernor::default() + }, + rent.clone(), // most tests don't expect rent + ClusterType::Development, + spl_programs(&rent), + ); + genesis_config.ticks_per_slot *= 8; + + // workaround for https://github.com/solana-labs/solana/issues/30085 + // the test can deploy and use spl_programs in the genensis slot without waiting for the next one + let (bank, bank_forks) = Bank::new_with_bank_forks_for_tests(&genesis_config); + + let bank = Arc::new(Bank::new_from_parent(bank, &Pubkey::default(), 1)); + + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Arc::new( + Blockstore::open(ledger_path.path()) + .expect("Expected to be able to open database ledger"), + ); + + let (exit, poh_recorder, poh_simulator, entry_receiver) = + create_test_recorder(&bank, blockstore, Some(PohConfig::default()), None); + + let validator_pubkey = voting_keypair.pubkey(); + TestFixture { + genesis_config_info: GenesisConfigInfo { + genesis_config, + mint_keypair, + voting_keypair, + validator_pubkey, + }, + leader_keypair, + bank, + bank_forks, + exit, + poh_recorder, + poh_simulator, + entry_receiver, + } + } + + fn make_random_overlapping_bundles( + mint_keypair: &Keypair, + num_bundles: usize, + num_packets_per_bundle: usize, + hash: Hash, + max_transfer_amount: u64, + ) -> Vec { + let mut rng = thread_rng(); + + (0..num_bundles) + .map(|_| { + let transfers: Vec<_> = (0..num_packets_per_bundle) + .map(|_| { + VersionedTransaction::from(transfer( + mint_keypair, + &mint_keypair.pubkey(), + rng.next_u64() % max_transfer_amount, + hash, + )) + }) + .collect(); + let bundle_id = derive_bundle_id(&transfers); + + PacketBundle { + batch: PacketBatch::new( + transfers + .iter() + .map(|tx| Packet::from_data(None, tx).unwrap()) + .collect(), + ), + bundle_id, + } + }) + .collect() + } + + fn get_tip_manager(vote_account: &Pubkey) -> TipManager { + TipManager::new(TipManagerConfig { + tip_payment_program_id: Pubkey::from_str("T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt") + .unwrap(), + tip_distribution_program_id: Pubkey::from_str( + "4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7", + ) + .unwrap(), + tip_distribution_account_config: TipDistributionAccountConfig { + merkle_root_upload_authority: Pubkey::new_unique(), + vote_account: *vote_account, + commission_bps: 10, + }, + }) + } + + /// Happy-path bundle execution w/ no tip management + #[test] + fn test_bundle_no_tip_success() { + solana_logger::setup(); + let TestFixture { + genesis_config_info, + leader_keypair, + bank, + exit, + poh_recorder, + poh_simulator, + entry_receiver, + bank_forks: _bank_forks, + } = create_test_fixture(1_000_000); + let recorder = poh_recorder.read().unwrap().new_recorder(); + + let status = poh_recorder + .read() + .unwrap() + .reached_leader_slot(&leader_keypair.pubkey()); + info!("status: {:?}", status); + + let (replay_vote_sender, _replay_vote_receiver) = unbounded(); + let committer = Committer::new( + None, + replay_vote_sender, + Arc::new(PrioritizationFeeCache::new(0u64)), + ); + + let block_builder_pubkey = Pubkey::new_unique(); + let tip_manager = get_tip_manager(&genesis_config_info.voting_keypair.pubkey()); + let block_builder_info = Arc::new(Mutex::new(BlockBuilderFeeInfo { + block_builder: block_builder_pubkey, + block_builder_commission: 10, + })); + + let cluster_info = Arc::new(ClusterInfo::new( + ContactInfo::new(leader_keypair.pubkey(), 0, 0), + Arc::new(leader_keypair), + SocketAddrSpace::new(true), + )); + + let mut consumer = BundleConsumer::new( + committer, + recorder, + QosService::new(1), + None, + tip_manager, + BundleAccountLocker::default(), + block_builder_info, + Duration::from_secs(10), + cluster_info, + BundleReservedSpaceManager::new( + MAX_BLOCK_UNITS, + 3_000_000, + poh_recorder + .read() + .unwrap() + .ticks_per_slot() + .saturating_mul(8) + .saturating_div(10), + ), + ); + + let bank_start = poh_recorder.read().unwrap().bank_start().unwrap(); + + let mut bundle_storage = UnprocessedTransactionStorage::new_bundle_storage(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(1); + + let mut packet_bundles = make_random_overlapping_bundles( + &genesis_config_info.mint_keypair, + 1, + 3, + genesis_config_info.genesis_config.hash(), + 10_000, + ); + let deserialized_bundle = BundlePacketDeserializer::deserialize_bundle( + packet_bundles.get_mut(0).unwrap(), + false, + None, + ) + .unwrap(); + let mut error_metrics = TransactionErrorMetrics::default(); + let sanitized_bundle = deserialized_bundle + .build_sanitized_bundle( + &bank_start.working_bank, + &HashSet::default(), + &mut error_metrics, + ) + .unwrap(); + + let summary = bundle_storage.insert_bundles(vec![deserialized_bundle]); + assert_eq!( + summary.num_packets_inserted, + sanitized_bundle.transactions.len() + ); + assert_eq!(summary.num_bundles_dropped, 0); + assert_eq!(summary.num_bundles_inserted, 1); + + consumer.consume_buffered_bundles( + &bank_start, + &mut bundle_storage, + &mut bundle_stage_leader_metrics, + ); + + let mut transactions = Vec::new(); + while let Ok(WorkingBankEntry { + bank: wbe_bank, + entries_ticks, + }) = entry_receiver.recv() + { + assert_eq!(bank.slot(), wbe_bank.slot()); + for (entry, _) in entries_ticks { + if !entry.transactions.is_empty() { + // transactions in this test are all overlapping, so each entry will contain 1 transaction + assert_eq!(entry.transactions.len(), 1); + transactions.extend(entry.transactions); + } + } + if transactions.len() == sanitized_bundle.transactions.len() { + break; + } + } + + let bundle_versioned_transactions: Vec<_> = sanitized_bundle + .transactions + .iter() + .map(|tx| tx.to_versioned_transaction()) + .collect(); + assert_eq!(transactions, bundle_versioned_transactions); + + let check_results = bank.check_transactions( + &sanitized_bundle.transactions, + &vec![Ok(()); sanitized_bundle.transactions.len()], + MAX_PROCESSING_AGE, + &mut error_metrics, + ); + + let expected_result: Vec = + vec![Err(TransactionError::AlreadyProcessed); sanitized_bundle.transactions.len()]; + + assert_eq!(check_results, expected_result); + + poh_recorder + .write() + .unwrap() + .is_exited + .store(true, Ordering::Relaxed); + exit.store(true, Ordering::Relaxed); + poh_simulator.join().unwrap(); + // TODO (LB): cleanup blockstore + } + + /// Happy-path bundle execution to ensure tip management works. + /// Tip management involves cranking setup bundles before executing the test bundle + #[test] + fn test_bundle_tip_program_setup_success() { + solana_logger::setup(); + let TestFixture { + genesis_config_info, + leader_keypair, + bank, + exit, + poh_recorder, + poh_simulator, + entry_receiver, + bank_forks: _bank_forks, + } = create_test_fixture(1_000_000); + let recorder = poh_recorder.read().unwrap().new_recorder(); + + let (replay_vote_sender, _replay_vote_receiver) = unbounded(); + let committer = Committer::new( + None, + replay_vote_sender, + Arc::new(PrioritizationFeeCache::new(0u64)), + ); + + let block_builder_pubkey = Pubkey::new_unique(); + let tip_manager = get_tip_manager(&genesis_config_info.voting_keypair.pubkey()); + let block_builder_info = Arc::new(Mutex::new(BlockBuilderFeeInfo { + block_builder: block_builder_pubkey, + block_builder_commission: 10, + })); + + let cluster_info = Arc::new(ClusterInfo::new( + ContactInfo::new(leader_keypair.pubkey(), 0, 0), + Arc::new(leader_keypair), + SocketAddrSpace::new(true), + )); + + let mut consumer = BundleConsumer::new( + committer, + recorder, + QosService::new(1), + None, + tip_manager.clone(), + BundleAccountLocker::default(), + block_builder_info, + Duration::from_secs(10), + cluster_info.clone(), + BundleReservedSpaceManager::new( + MAX_BLOCK_UNITS, + 3_000_000, + poh_recorder + .read() + .unwrap() + .ticks_per_slot() + .saturating_mul(8) + .saturating_div(10), + ), + ); + + let bank_start = poh_recorder.read().unwrap().bank_start().unwrap(); + + let mut bundle_storage = UnprocessedTransactionStorage::new_bundle_storage(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(1); + // MAIN LOGIC + + // a bundle that tips the tip program + let tip_accounts = tip_manager.get_tip_accounts(); + let tip_account = tip_accounts.iter().collect::>()[0]; + let mut packet_bundle = PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data( + None, + transfer( + &genesis_config_info.mint_keypair, + tip_account, + 1, + genesis_config_info.genesis_config.hash(), + ), + ) + .unwrap()]), + bundle_id: "test_transfer".to_string(), + }; + + let deserialized_bundle = + BundlePacketDeserializer::deserialize_bundle(&mut packet_bundle, false, None).unwrap(); + let mut error_metrics = TransactionErrorMetrics::default(); + let sanitized_bundle = deserialized_bundle + .build_sanitized_bundle( + &bank_start.working_bank, + &HashSet::default(), + &mut error_metrics, + ) + .unwrap(); + + let summary = bundle_storage.insert_bundles(vec![deserialized_bundle]); + assert_eq!(summary.num_bundles_inserted, 1); + assert_eq!(summary.num_packets_inserted, 1); + assert_eq!(summary.num_bundles_dropped, 0); + + consumer.consume_buffered_bundles( + &bank_start, + &mut bundle_storage, + &mut bundle_stage_leader_metrics, + ); + + // its expected there are 3 transactions. One to initialize the tip program configuration, one to change the tip receiver, + // and another with the tip + + let mut transactions = Vec::new(); + while let Ok(WorkingBankEntry { + bank: wbe_bank, + entries_ticks, + }) = entry_receiver.recv() + { + assert_eq!(bank.slot(), wbe_bank.slot()); + transactions.extend(entries_ticks.into_iter().flat_map(|(e, _)| e.transactions)); + if transactions.len() == 5 { + break; + } + } + + // tip management on the first bundle involves: + // calling initialize on the tip payment and tip distribution programs + // creating the tip distribution account for this validator's epoch (the MEV piggy bank) + // changing the tip receiver and block builder tx + // the original transfer that was sent + let keypair = cluster_info.keypair().clone(); + + assert_eq!( + transactions[0], + tip_manager + .initialize_tip_payment_program_tx(&bank, &keypair) + .to_versioned_transaction() + ); + assert_eq!( + transactions[1], + tip_manager + .initialize_tip_distribution_config_tx(&bank, &keypair) + .to_versioned_transaction() + ); + assert_eq!( + transactions[2], + tip_manager + .initialize_tip_distribution_account_tx(&bank, &keypair) + .to_versioned_transaction() + ); + // the first tip receiver + block builder are the initializer (keypair.pubkey()) as set by the + // TipPayment program during initialization + assert_eq!( + transactions[3], + tip_manager + .build_change_tip_receiver_and_block_builder_tx( + &keypair.pubkey(), + &derive_tip_distribution_account_address( + &tip_manager.tip_distribution_program_id(), + &genesis_config_info.validator_pubkey, + bank_start.working_bank.epoch() + ) + .0, + &bank_start.working_bank, + &keypair, + &keypair.pubkey(), + &block_builder_pubkey, + 10 + ) + .to_versioned_transaction() + ); + assert_eq!( + transactions[4], + sanitized_bundle.transactions[0].to_versioned_transaction() + ); + + poh_recorder + .write() + .unwrap() + .is_exited + .store(true, Ordering::Relaxed); + exit.store(true, Ordering::Relaxed); + poh_simulator.join().unwrap(); + } + + #[test] + fn test_handle_tip_programs() { + solana_logger::setup(); + let TestFixture { + genesis_config_info, + leader_keypair, + bank, + exit, + poh_recorder, + poh_simulator, + entry_receiver, + bank_forks: _bank_forks, + } = create_test_fixture(1_000_000); + let recorder = poh_recorder.read().unwrap().new_recorder(); + + let (replay_vote_sender, _replay_vote_receiver) = unbounded(); + let committer = Committer::new( + None, + replay_vote_sender, + Arc::new(PrioritizationFeeCache::new(0u64)), + ); + + let block_builder_pubkey = Pubkey::new_unique(); + let tip_manager = get_tip_manager(&genesis_config_info.voting_keypair.pubkey()); + let block_builder_info = Arc::new(Mutex::new(BlockBuilderFeeInfo { + block_builder: block_builder_pubkey, + block_builder_commission: 10, + })); + + let cluster_info = Arc::new(ClusterInfo::new( + ContactInfo::new(leader_keypair.pubkey(), 0, 0), + Arc::new(leader_keypair), + SocketAddrSpace::new(true), + )); + + let bank_start = poh_recorder.read().unwrap().bank_start().unwrap(); + + let reserved_ticks = bank.max_tick_height().saturating_mul(8).saturating_div(10); + + // The first 80% of the block, based on poh ticks, has `preallocated_bundle_cost` less compute units. + // The last 20% has has full compute so blockspace is maximized if BundleStage is idle. + let reserved_space = + BundleReservedSpaceManager::new(MAX_BLOCK_UNITS, 3_000_000, reserved_ticks); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(1); + assert_matches!( + BundleConsumer::handle_tip_programs( + &BundleAccountLocker::default(), + &tip_manager, + &cluster_info, + &block_builder_info, + &committer, + &recorder, + &QosService::new(1), + &None, + Duration::from_secs(10), + &reserved_space, + &bank_start, + &mut bundle_stage_leader_metrics + ), + Ok(()) + ); + + let mut transactions = Vec::new(); + while let Ok(WorkingBankEntry { + bank: wbe_bank, + entries_ticks, + }) = entry_receiver.recv() + { + assert_eq!(bank.slot(), wbe_bank.slot()); + transactions.extend(entries_ticks.into_iter().flat_map(|(e, _)| e.transactions)); + if transactions.len() == 4 { + break; + } + } + + let keypair = cluster_info.keypair().clone(); + // expect to see initialize tip payment program, tip distribution program, initialize tip distribution account, change tip receiver + change block builder + assert_eq!( + transactions[0], + tip_manager + .initialize_tip_payment_program_tx(&bank, &keypair) + .to_versioned_transaction() + ); + assert_eq!( + transactions[1], + tip_manager + .initialize_tip_distribution_config_tx(&bank, &keypair) + .to_versioned_transaction() + ); + assert_eq!( + transactions[2], + tip_manager + .initialize_tip_distribution_account_tx(&bank, &keypair) + .to_versioned_transaction() + ); + // the first tip receiver + block builder are the initializer (keypair.pubkey()) as set by the + // TipPayment program during initialization + assert_eq!( + transactions[3], + tip_manager + .build_change_tip_receiver_and_block_builder_tx( + &keypair.pubkey(), + &derive_tip_distribution_account_address( + &tip_manager.tip_distribution_program_id(), + &genesis_config_info.validator_pubkey, + bank_start.working_bank.epoch() + ) + .0, + &bank_start.working_bank, + &keypair, + &keypair.pubkey(), + &block_builder_pubkey, + 10 + ) + .to_versioned_transaction() + ); + + poh_recorder + .write() + .unwrap() + .is_exited + .store(true, Ordering::Relaxed); + exit.store(true, Ordering::Relaxed); + poh_simulator.join().unwrap(); + } + + #[test] + fn test_reserve_bundle_blockspace_success() { + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10); + let bank = Arc::new(Bank::new_for_tests(&genesis_config)); + + let keypair1 = Keypair::new(); + let keypair2 = Keypair::new(); + let transfer_tx = SanitizedTransaction::from_transaction_for_tests(transfer( + &keypair1, + &keypair2.pubkey(), + 1, + bank.parent_hash(), + )); + let sanitized_bundle = SanitizedBundle { + transactions: vec![transfer_tx], + bundle_id: String::default(), + }; + + let transfer_cost = + CostModel::calculate_cost(&sanitized_bundle.transactions[0], &bank.feature_set); + + let qos_service = QosService::new(1); + let reserved_ticks = bank.max_tick_height().saturating_mul(8).saturating_div(10); + + // The first 80% of the block, based on poh ticks, has `preallocated_bundle_cost` less compute units. + // The last 20% has has full compute so blockspace is maximized if BundleStage is idle. + let reserved_space = + BundleReservedSpaceManager::new(MAX_BLOCK_UNITS, 3_000_000, reserved_ticks); + + assert!(BundleConsumer::reserve_bundle_blockspace( + &qos_service, + &reserved_space, + &sanitized_bundle, + &bank + ) + .is_ok()); + assert_eq!( + bank.read_cost_tracker().unwrap().block_cost(), + transfer_cost.sum() + ); + } + + #[test] + fn test_reserve_bundle_blockspace_failure() { + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10); + let bank = Arc::new(Bank::new_for_tests(&genesis_config)); + + let keypair1 = Keypair::new(); + let keypair2 = Keypair::new(); + let transfer_tx1 = SanitizedTransaction::from_transaction_for_tests(transfer( + &keypair1, + &keypair2.pubkey(), + 1, + bank.parent_hash(), + )); + let transfer_tx2 = SanitizedTransaction::from_transaction_for_tests(transfer( + &keypair1, + &keypair2.pubkey(), + 2, + bank.parent_hash(), + )); + let sanitized_bundle = SanitizedBundle { + transactions: vec![transfer_tx1, transfer_tx2], + bundle_id: String::default(), + }; + + // set block cost limit to 1 transfer transaction, try to process 2, should return an error + // and rollback block cost added + let transfer_cost = + CostModel::calculate_cost(&sanitized_bundle.transactions[0], &bank.feature_set); + bank.write_cost_tracker() + .unwrap() + .set_block_cost_limit(transfer_cost.sum()); + + let qos_service = QosService::new(1); + let reserved_ticks = bank.max_tick_height().saturating_mul(8).saturating_div(10); + + // The first 80% of the block, based on poh ticks, has `preallocated_bundle_cost` less compute units. + // The last 20% has has full compute so blockspace is maximized if BundleStage is idle. + let reserved_space = BundleReservedSpaceManager::new( + bank.read_cost_tracker().unwrap().block_cost(), + 50, + reserved_ticks, + ); + + assert!(BundleConsumer::reserve_bundle_blockspace( + &qos_service, + &reserved_space, + &sanitized_bundle, + &bank + ) + .is_err()); + assert_eq!(bank.read_cost_tracker().unwrap().block_cost(), 0); + assert_eq!( + bank.read_cost_tracker().unwrap().block_cost_limit(), + bank.read_cost_tracker() + .unwrap() + .block_cost_limit() + .saturating_sub(50) + ); + } +} diff --git a/core/src/bundle_stage/bundle_packet_deserializer.rs b/core/src/bundle_stage/bundle_packet_deserializer.rs new file mode 100644 index 0000000000..563902722e --- /dev/null +++ b/core/src/bundle_stage/bundle_packet_deserializer.rs @@ -0,0 +1,271 @@ +//! Deserializes PacketBundles +use { + crate::{ + immutable_deserialized_bundle::{DeserializedBundleError, ImmutableDeserializedBundle}, + packet_bundle::PacketBundle, + }, + crossbeam_channel::{Receiver, RecvTimeoutError}, + solana_runtime::bank_forks::BankForks, + solana_sdk::saturating_add_assign, + std::{ + sync::{Arc, RwLock}, + time::{Duration, Instant}, + }, +}; + +/// Results from deserializing packet batches. +#[derive(Debug)] +pub struct ReceiveBundleResults { + /// Deserialized bundles from all received bundle packets + pub deserialized_bundles: Vec, + /// Number of dropped bundles + pub num_dropped_bundles: usize, +} + +pub struct BundlePacketDeserializer { + /// Receiver for bundle packets + bundle_packet_receiver: Receiver>, + /// Provides working bank for deserializer to check feature activation + bank_forks: Arc>, + /// Max packets per bundle + max_packets_per_bundle: Option, +} + +impl BundlePacketDeserializer { + pub fn new( + bundle_packet_receiver: Receiver>, + bank_forks: Arc>, + max_packets_per_bundle: Option, + ) -> Self { + Self { + bundle_packet_receiver, + bank_forks, + max_packets_per_bundle, + } + } + + /// Handles receiving bundles and deserializing them + pub fn receive_bundles( + &self, + recv_timeout: Duration, + capacity: usize, + ) -> Result { + let (bundle_count, _packet_count, mut bundles) = + self.receive_until(recv_timeout, capacity)?; + + // Note: this can be removed after feature `round_compute_unit_price` is activated in + // mainnet-beta + let _working_bank = self.bank_forks.read().unwrap().working_bank(); + let round_compute_unit_price_enabled = false; // TODO get from working_bank.feature_set + + Ok(Self::deserialize_and_collect_bundles( + bundle_count, + &mut bundles, + round_compute_unit_price_enabled, + self.max_packets_per_bundle, + )) + } + + /// Deserialize packet batches, aggregates tracer packet stats, and collect + /// them into ReceivePacketResults + fn deserialize_and_collect_bundles( + bundle_count: usize, + bundles: &mut [PacketBundle], + round_compute_unit_price_enabled: bool, + max_packets_per_bundle: Option, + ) -> ReceiveBundleResults { + let mut deserialized_bundles = Vec::with_capacity(bundle_count); + let mut num_dropped_bundles: usize = 0; + + for bundle in bundles.iter_mut() { + match Self::deserialize_bundle( + bundle, + round_compute_unit_price_enabled, + max_packets_per_bundle, + ) { + Ok(deserialized_bundle) => { + deserialized_bundles.push(deserialized_bundle); + } + Err(_) => { + saturating_add_assign!(num_dropped_bundles, 1); + } + } + } + + ReceiveBundleResults { + deserialized_bundles, + num_dropped_bundles, + } + } + + /// Receives bundle packets + fn receive_until( + &self, + recv_timeout: Duration, + bundle_count_upperbound: usize, + ) -> Result<(usize, usize, Vec), RecvTimeoutError> { + let start = Instant::now(); + + let mut bundles = self.bundle_packet_receiver.recv_timeout(recv_timeout)?; + let mut num_packets_received: usize = bundles.iter().map(|pb| pb.batch.len()).sum(); + let mut num_bundles_received: usize = bundles.len(); + + if num_bundles_received <= bundle_count_upperbound { + while let Ok(bundle_packets) = self.bundle_packet_receiver.try_recv() { + trace!("got more packet batches in bundle packet deserializer"); + + saturating_add_assign!( + num_packets_received, + bundle_packets + .iter() + .map(|pb| pb.batch.len()) + .sum::() + ); + saturating_add_assign!(num_bundles_received, bundle_packets.len()); + + bundles.extend(bundle_packets); + + if start.elapsed() >= recv_timeout + || num_bundles_received >= bundle_count_upperbound + { + break; + } + } + } + + Ok((num_bundles_received, num_packets_received, bundles)) + } + + /// Deserializes the Bundle into DeserializedBundlePackets, returning None if any packet in the + /// bundle failed to deserialize + pub fn deserialize_bundle( + bundle: &mut PacketBundle, + round_compute_unit_price_enabled: bool, + max_packets_per_bundle: Option, + ) -> Result { + bundle.batch.iter_mut().for_each(|p| { + p.meta_mut() + .set_round_compute_unit_price(round_compute_unit_price_enabled); + }); + + ImmutableDeserializedBundle::new(bundle, max_packets_per_bundle) + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + crossbeam_channel::unbounded, + solana_ledger::genesis_utils::create_genesis_config, + solana_perf::packet::PacketBatch, + solana_runtime::{bank::Bank, genesis_utils::GenesisConfigInfo}, + solana_sdk::{packet::Packet, signature::Signer, system_transaction::transfer}, + }; + + #[test] + fn test_deserialize_and_collect_bundles_empty() { + let results = + BundlePacketDeserializer::deserialize_and_collect_bundles(0, &mut [], false, Some(5)); + assert_eq!(results.deserialized_bundles.len(), 0); + assert_eq!(results.num_dropped_bundles, 0); + } + + #[test] + fn test_receive_bundles_capacity() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let (sender, receiver) = unbounded(); + + let deserializer = BundlePacketDeserializer::new(receiver, bank_forks, Some(10)); + + let packet_bundles: Vec<_> = (0..10) + .map(|_| PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data( + None, + transfer( + &mint_keypair, + &mint_keypair.pubkey(), + 100, + genesis_config.hash(), + ), + ) + .unwrap()]), + bundle_id: String::default(), + }) + .collect(); + + sender.send(packet_bundles.clone()).unwrap(); + + let bundles = deserializer + .receive_bundles(Duration::from_millis(100), 5) + .unwrap(); + // this is confusing, but it's sent as one batch + assert_eq!(bundles.deserialized_bundles.len(), 10); + assert_eq!(bundles.num_dropped_bundles, 0); + + // make sure empty + assert_matches!( + deserializer.receive_bundles(Duration::from_millis(100), 5), + Err(RecvTimeoutError::Timeout) + ); + + // send 2x 10 size batches. capacity is 5, but will return 10 since that's the batch size + sender.send(packet_bundles.clone()).unwrap(); + sender.send(packet_bundles).unwrap(); + let bundles = deserializer + .receive_bundles(Duration::from_millis(100), 5) + .unwrap(); + assert_eq!(bundles.deserialized_bundles.len(), 10); + assert_eq!(bundles.num_dropped_bundles, 0); + + let bundles = deserializer + .receive_bundles(Duration::from_millis(100), 5) + .unwrap(); + assert_eq!(bundles.deserialized_bundles.len(), 10); + assert_eq!(bundles.num_dropped_bundles, 0); + + assert_matches!( + deserializer.receive_bundles(Duration::from_millis(100), 5), + Err(RecvTimeoutError::Timeout) + ); + } + + #[test] + fn test_receive_bundles_bad_bundles() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair: _, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let (sender, receiver) = unbounded(); + + let deserializer = BundlePacketDeserializer::new(receiver, bank_forks, Some(10)); + + let packet_bundles: Vec<_> = (0..10) + .map(|_| PacketBundle { + batch: PacketBatch::new(vec![]), + bundle_id: String::default(), + }) + .collect(); + sender.send(packet_bundles).unwrap(); + + let bundles = deserializer + .receive_bundles(Duration::from_millis(100), 5) + .unwrap(); + // this is confusing, but it's sent as one batch + assert_eq!(bundles.deserialized_bundles.len(), 0); + assert_eq!(bundles.num_dropped_bundles, 10); + } +} diff --git a/core/src/bundle_stage/bundle_packet_receiver.rs b/core/src/bundle_stage/bundle_packet_receiver.rs new file mode 100644 index 0000000000..c99d9aa020 --- /dev/null +++ b/core/src/bundle_stage/bundle_packet_receiver.rs @@ -0,0 +1,825 @@ +use { + super::BundleStageLoopMetrics, + crate::{ + banking_stage::unprocessed_transaction_storage::UnprocessedTransactionStorage, + bundle_stage::{ + bundle_packet_deserializer::{BundlePacketDeserializer, ReceiveBundleResults}, + bundle_stage_leader_metrics::BundleStageLeaderMetrics, + }, + immutable_deserialized_bundle::ImmutableDeserializedBundle, + packet_bundle::PacketBundle, + }, + crossbeam_channel::{Receiver, RecvTimeoutError}, + solana_measure::{measure::Measure, measure_us}, + solana_runtime::bank_forks::BankForks, + solana_sdk::timing::timestamp, + std::{ + sync::{Arc, RwLock}, + time::Duration, + }, +}; + +pub struct BundleReceiver { + id: u32, + bundle_packet_deserializer: BundlePacketDeserializer, +} + +impl BundleReceiver { + pub fn new( + id: u32, + bundle_packet_receiver: Receiver>, + bank_forks: Arc>, + max_packets_per_bundle: Option, + ) -> Self { + Self { + id, + bundle_packet_deserializer: BundlePacketDeserializer::new( + bundle_packet_receiver, + bank_forks, + max_packets_per_bundle, + ), + } + } + + /// Receive incoming packets, push into unprocessed buffer with packet indexes + pub fn receive_and_buffer_bundles( + &mut self, + unprocessed_bundle_storage: &mut UnprocessedTransactionStorage, + bundle_stage_metrics: &mut BundleStageLoopMetrics, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) -> Result<(), RecvTimeoutError> { + let (result, recv_time_us) = measure_us!({ + let recv_timeout = Self::get_receive_timeout(unprocessed_bundle_storage); + let mut recv_and_buffer_measure = Measure::start("recv_and_buffer"); + self.bundle_packet_deserializer + .receive_bundles(recv_timeout, unprocessed_bundle_storage.max_receive_size()) + // Consumes results if Ok, otherwise we keep the Err + .map(|receive_bundle_results| { + self.buffer_bundles( + receive_bundle_results, + unprocessed_bundle_storage, + bundle_stage_metrics, + // tracer_packet_stats, + bundle_stage_leader_metrics, + ); + recv_and_buffer_measure.stop(); + bundle_stage_metrics.increment_receive_and_buffer_bundles_elapsed_us( + recv_and_buffer_measure.as_us(), + ); + }) + }); + + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_receive_and_buffer_packets_us(recv_time_us); + + result + } + + fn get_receive_timeout( + unprocessed_transaction_storage: &UnprocessedTransactionStorage, + ) -> Duration { + // Gossip thread will almost always not wait because the transaction storage will most likely not be empty + if !unprocessed_transaction_storage.is_empty() { + // If there are buffered packets, run the equivalent of try_recv to try reading more + // packets. This prevents starving BankingStage::consume_buffered_packets due to + // buffered_packet_batches containing transactions that exceed the cost model for + // the current bank. + Duration::from_millis(0) + } else { + // BundleStage should pick up a working_bank as fast as possible + Duration::from_millis(100) + } + } + + fn buffer_bundles( + &self, + ReceiveBundleResults { + deserialized_bundles, + num_dropped_bundles, + }: ReceiveBundleResults, + unprocessed_transaction_storage: &mut UnprocessedTransactionStorage, + bundle_stage_stats: &mut BundleStageLoopMetrics, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + ) { + let bundle_count = deserialized_bundles.len(); + let packet_count: usize = deserialized_bundles.iter().map(|b| b.len()).sum(); + + bundle_stage_stats.increment_num_bundles_received(bundle_count as u64); + bundle_stage_stats.increment_num_packets_received(packet_count as u64); + bundle_stage_stats.increment_num_bundles_dropped(num_dropped_bundles as u64); + // TODO (LB): fix this + // bundle_stage_leader_metrics + // .leader_slot_metrics_tracker() + // .increment_total_new_valid_packets(packet_count as u64); + + debug!( + "@{:?} bundles: {} txs: {} id: {}", + timestamp(), + bundle_count, + packet_count, + self.id + ); + + Self::push_unprocessed( + unprocessed_transaction_storage, + deserialized_bundles, + bundle_stage_leader_metrics, + bundle_stage_stats, + ); + } + + fn push_unprocessed( + unprocessed_transaction_storage: &mut UnprocessedTransactionStorage, + deserialized_bundles: Vec, + bundle_stage_leader_metrics: &mut BundleStageLeaderMetrics, + bundle_stage_stats: &mut BundleStageLoopMetrics, + ) { + if !deserialized_bundles.is_empty() { + let insert_bundles_summary = + unprocessed_transaction_storage.insert_bundles(deserialized_bundles); + + bundle_stage_stats.increment_newly_buffered_bundles_count( + insert_bundles_summary.num_bundles_inserted as u64, + ); + bundle_stage_stats + .increment_num_bundles_dropped(insert_bundles_summary.num_bundles_dropped as u64); + + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .increment_newly_buffered_packets_count( + insert_bundles_summary.num_packets_inserted as u64, + ); + + bundle_stage_leader_metrics + .leader_slot_metrics_tracker() + .accumulate_insert_packet_batches_summary( + &insert_bundles_summary.insert_packets_summary, + ); + } + } +} + +/// This tests functionality of BundlePacketReceiver and the internals of BundleStorage because +/// they're tightly intertwined +#[cfg(test)] +mod tests { + use { + super::*, + crate::banking_stage::unprocessed_transaction_storage::BundleStorage, + crossbeam_channel::unbounded, + rand::{thread_rng, RngCore}, + solana_bundle::{ + bundle_execution::LoadAndExecuteBundleError, BundleExecutionError, TipError, + }, + solana_ledger::genesis_utils::create_genesis_config, + solana_perf::packet::PacketBatch, + solana_poh::poh_recorder::PohRecorderError, + solana_runtime::{bank::Bank, genesis_utils::GenesisConfigInfo}, + solana_sdk::{ + bundle::{derive_bundle_id, SanitizedBundle}, + hash::Hash, + packet::Packet, + signature::{Keypair, Signer}, + system_transaction::transfer, + transaction::VersionedTransaction, + }, + std::collections::HashSet, + }; + + /// Makes `num_bundles` random bundles with `num_packets_per_bundle` packets per bundle. + fn make_random_bundles( + mint_keypair: &Keypair, + num_bundles: usize, + num_packets_per_bundle: usize, + hash: Hash, + ) -> Vec { + let mut rng = thread_rng(); + + (0..num_bundles) + .map(|_| { + let transfers: Vec<_> = (0..num_packets_per_bundle) + .map(|_| { + VersionedTransaction::from(transfer( + mint_keypair, + &mint_keypair.pubkey(), + rng.next_u64(), + hash, + )) + }) + .collect(); + let bundle_id = derive_bundle_id(&transfers); + + PacketBundle { + batch: PacketBatch::new( + transfers + .iter() + .map(|tx| Packet::from_data(None, tx).unwrap()) + .collect(), + ), + bundle_id, + } + }) + .collect() + } + + fn assert_bundles_same( + packet_bundles: &[PacketBundle], + bundles_to_process: &[(ImmutableDeserializedBundle, SanitizedBundle)], + ) { + assert_eq!(packet_bundles.len(), bundles_to_process.len()); + packet_bundles + .iter() + .zip(bundles_to_process.iter()) + .for_each(|(packet_bundle, (_, sanitized_bundle))| { + assert_eq!(packet_bundle.bundle_id, sanitized_bundle.bundle_id); + assert_eq!( + packet_bundle.batch.len(), + sanitized_bundle.transactions.len() + ); + }); + } + + #[test] + fn test_receive_bundles() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + let bundles = make_random_bundles(&mint_keypair, 10, 2, genesis_config.hash()); + sender.send(bundles.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 10); + assert_eq!(bundle_storage.unprocessed_packets_len(), 20); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_packets_len(), 0); + assert_eq!(bundle_storage.max_receive_size(), 990); + + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles, bundles_to_process); + (0..bundles_to_process.len()).map(|_| Ok(())).collect() + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.unprocessed_packets_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_packets_len(), 0); + assert_eq!(bundle_storage.max_receive_size(), 1000); + } + + #[test] + fn test_receive_more_bundles_than_capacity() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 5 more than capacity + let bundles = make_random_bundles( + &mint_keypair, + BundleStorage::BUNDLE_STORAGE_CAPACITY + 5, + 2, + genesis_config.hash(), + ); + + sender.send(bundles.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + // 1005 bundles were sent, but the capacity is 1000 + assert_eq!(bundle_storage.unprocessed_bundles_len(), 1000); + assert_eq!(bundle_storage.unprocessed_packets_len(), 2000); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_packets_len(), 0); + + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + // make sure the first 1000 bundles are the ones to process + assert_bundles_same( + &bundles[0..BundleStorage::BUNDLE_STORAGE_CAPACITY], + bundles_to_process, + ); + (0..bundles_to_process.len()).map(|_| Ok(())).collect() + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + } + + #[test] + fn test_process_bundles_poh_record_error_rebuffered() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 5 bundles across the queue + let bundles = make_random_bundles(&mint_keypair, 5, 2, genesis_config.hash()); + sender.send(bundles.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let poh_max_height_reached_index = 3; + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + + // make sure poh end of slot reached + the correct bundles are buffered for the next time. + // bundles at index 3 + 4 are rebuffered + assert!(bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles, bundles_to_process); + + let mut results = vec![Ok(()); bundles_to_process.len()]; + + (poh_max_height_reached_index..bundles_to_process.len()).for_each(|index| { + results[index] = Err(BundleExecutionError::PohRecordError( + PohRecorderError::MaxHeightReached, + )); + }); + results + } + )); + + assert_eq!(bundle_storage.unprocessed_bundles_len(), 2); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles[poh_max_height_reached_index..], bundles_to_process); + vec![Ok(()); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + } + + #[test] + fn test_process_bundles_bank_processing_done_rebuffered() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 5 bundles across the queue + let bundles = make_random_bundles(&mint_keypair, 5, 2, genesis_config.hash()); + sender.send(bundles.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bank_processing_done_index = 3; + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + + // bundles at index 3 + 4 are rebuffered + assert!(bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles, bundles_to_process); + + let mut results = vec![Ok(()); bundles_to_process.len()]; + + (bank_processing_done_index..bundles_to_process.len()).for_each(|index| { + results[index] = Err(BundleExecutionError::BankProcessingTimeLimitReached); + }); + results + } + )); + + // 0, 1, 2 processed; 3, 4 buffered + assert_eq!(bundle_storage.unprocessed_bundles_len(), 2); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles[bank_processing_done_index..], bundles_to_process); + vec![Ok(()); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + } + + #[test] + fn test_process_bundles_bank_execution_error_dropped() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 5 bundles across the queue + let bundles = make_random_bundles(&mint_keypair, 5, 2, genesis_config.hash()); + sender.send(bundles.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles, bundles_to_process); + vec![ + Err(BundleExecutionError::TransactionFailure( + LoadAndExecuteBundleError::ProcessingTimeExceeded(Duration::from_secs(1)), + )); + bundles_to_process.len() + ] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + } + + #[test] + fn test_process_bundles_tip_error_dropped() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 5 bundles across the queue + let bundles = make_random_bundles(&mint_keypair, 5, 2, genesis_config.hash()); + sender.send(bundles.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles, bundles_to_process); + vec![ + Err(BundleExecutionError::TipError(TipError::LockError)); + bundles_to_process.len() + ] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + } + + #[test] + fn test_process_bundles_lock_error_dropped() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 5 bundles across the queue + let bundles = make_random_bundles(&mint_keypair, 5, 2, genesis_config.hash()); + sender.send(bundles).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + vec![Err(BundleExecutionError::LockError); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + } + + #[test] + fn test_process_bundles_cost_model_exceeded_set_aside_and_requeued() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 5 bundles across the queue + let bundles = make_random_bundles(&mint_keypair, 5, 2, genesis_config.hash()); + sender.send(bundles.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + + // buffered bundles are moved to cost model side deque + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles, bundles_to_process); + vec![Err(BundleExecutionError::ExceedsCostModel); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 5); + + // double check there's no bundles to process + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert!(bundles_to_process.is_empty()); + vec![Ok(()); bundles_to_process.len()] + } + )); + + // create a new bank w/ new slot number, cost model buffered packets should move back onto queue + // in the same order they were originally + let bank = bank_forks.read().unwrap().working_bank(); + let new_bank = Arc::new(Bank::new_from_parent( + bank.clone(), + bank.collector_id(), + bank.slot() + 1, + )); + assert!(!bundle_storage.process_bundles( + new_bank, + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + // make sure same order as original + assert_bundles_same(&bundles, bundles_to_process); + vec![Ok(()); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + } + + #[test] + fn test_process_bundles_cost_model_exceeded_buffer_capacity() { + solana_logger::setup(); + + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (_, bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let mut unprocessed_storage = UnprocessedTransactionStorage::new_bundle_storage(); + + let (sender, receiver) = unbounded(); + let mut bundle_receiver = BundleReceiver::new(0, receiver, bank_forks.clone(), Some(5)); + + // send 500 bundles across the queue + let bundles0 = make_random_bundles( + &mint_keypair, + BundleStorage::BUNDLE_STORAGE_CAPACITY / 2, + 2, + genesis_config.hash(), + ); + sender.send(bundles0.clone()).unwrap(); + + let mut bundle_stage_stats = BundleStageLoopMetrics::default(); + let mut bundle_stage_leader_metrics = BundleStageLeaderMetrics::new(0); + + // receive and buffer bundles to the cost model reserve to test the capacity/dropped bundles there + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + // buffered bundles are moved to cost model side deque + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles0, bundles_to_process); + vec![Err(BundleExecutionError::ExceedsCostModel); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 500); + + let bundles1 = make_random_bundles( + &mint_keypair, + BundleStorage::BUNDLE_STORAGE_CAPACITY / 2, + 2, + genesis_config.hash(), + ); + sender.send(bundles1.clone()).unwrap(); + // should get 500 more bundles, cost model buffered length should be 1000 + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + // buffered bundles are moved to cost model side deque + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles1, bundles_to_process); + vec![Err(BundleExecutionError::ExceedsCostModel); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 1000); // full now + + // send 10 bundles to go over capacity + let bundles2 = make_random_bundles(&mint_keypair, 10, 2, genesis_config.hash()); + sender.send(bundles2.clone()).unwrap(); + + // this set will get dropped from cost model buffered bundles + let result = bundle_receiver.receive_and_buffer_bundles( + &mut unprocessed_storage, + &mut bundle_stage_stats, + &mut bundle_stage_leader_metrics, + ); + assert!(result.is_ok()); + + let bundle_storage = unprocessed_storage.bundle_storage().unwrap(); + // buffered bundles are moved to cost model side deque, but its at capacity so stays the same size + assert!(!bundle_storage.process_bundles( + bank_forks.read().unwrap().working_bank(), + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + assert_bundles_same(&bundles2, bundles_to_process); + vec![Err(BundleExecutionError::ExceedsCostModel); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 1000); + + // create new bank then call process_bundles again, expect to see [bundles1,bundles2] + let bank = bank_forks.read().unwrap().working_bank(); + let new_bank = Arc::new(Bank::new_from_parent( + bank.clone(), + bank.collector_id(), + bank.slot() + 1, + )); + assert!(!bundle_storage.process_bundles( + new_bank, + &mut bundle_stage_leader_metrics, + &HashSet::default(), + |bundles_to_process, _stats| { + // make sure same order as original + let expected_bundles: Vec<_> = + bundles0.iter().chain(bundles1.iter()).cloned().collect(); + assert_bundles_same(&expected_bundles, bundles_to_process); + vec![Ok(()); bundles_to_process.len()] + } + )); + assert_eq!(bundle_storage.unprocessed_bundles_len(), 0); + assert_eq!(bundle_storage.cost_model_buffered_bundles_len(), 0); + } +} diff --git a/core/src/bundle_stage/bundle_reserved_space_manager.rs b/core/src/bundle_stage/bundle_reserved_space_manager.rs new file mode 100644 index 0000000000..24cca76aa1 --- /dev/null +++ b/core/src/bundle_stage/bundle_reserved_space_manager.rs @@ -0,0 +1,237 @@ +use {solana_runtime::bank::Bank, solana_sdk::clock::Slot, std::sync::Arc}; + +/// Manager responsible for reserving `bundle_reserved_cost` during the first `reserved_ticks` of a bank +/// and resetting the block cost limit to `block_cost_limit` after the reserved tick period is over +pub struct BundleReservedSpaceManager { + // the bank's cost limit + block_cost_limit: u64, + // bundles get this much reserved space for the first reserved_ticks + bundle_reserved_cost: u64, + // a reduced block_compute_limit is reserved for this many ticks, afterwards it goes back to full cost + reserved_ticks: u64, + last_slot_updated: Slot, +} + +impl BundleReservedSpaceManager { + pub fn new(block_cost_limit: u64, bundle_reserved_cost: u64, reserved_ticks: u64) -> Self { + Self { + block_cost_limit, + bundle_reserved_cost, + reserved_ticks, + last_slot_updated: u64::MAX, + } + } + + /// Call this on creation of new bank and periodically while bundle processing + /// to manage the block_cost_limits + pub fn tick(&mut self, bank: &Arc) { + if self.last_slot_updated == bank.slot() && !self.is_in_reserved_tick_period(bank) { + // new slot logic already ran, need to revert the block cost limit to original if + // ticks are past the reserved tick mark + debug!( + "slot: {} ticks: {}, resetting block_cost_limit to {}", + bank.slot(), + bank.tick_height(), + self.block_cost_limit + ); + bank.write_cost_tracker() + .unwrap() + .set_block_cost_limit(self.block_cost_limit); + } else if self.last_slot_updated != bank.slot() && self.is_in_reserved_tick_period(bank) { + // new slot, if in the first max_tick - tick_height slots reserve space + // otherwise can leave the current block limit as is + let new_block_cost_limit = self.reduced_block_cost_limit(); + debug!( + "slot: {} ticks: {}, reserving block_cost_limit with block_cost_limit of {}", + bank.slot(), + bank.tick_height(), + new_block_cost_limit + ); + bank.write_cost_tracker() + .unwrap() + .set_block_cost_limit(new_block_cost_limit); + self.last_slot_updated = bank.slot(); + } + } + + /// return true if the bank is still in the period where block_cost_limits is reduced + pub fn is_in_reserved_tick_period(&self, bank: &Bank) -> bool { + bank.tick_height() % bank.ticks_per_slot() < self.reserved_ticks + } + + /// return the block_cost_limits as determined by the tick height of the bank + pub fn expected_block_cost_limits(&self, bank: &Bank) -> u64 { + if self.is_in_reserved_tick_period(bank) { + self.reduced_block_cost_limit() + } else { + self.block_cost_limit() + } + } + + pub fn reduced_block_cost_limit(&self) -> u64 { + self.block_cost_limit + .saturating_sub(self.bundle_reserved_cost) + } + + pub fn block_cost_limit(&self) -> u64 { + self.block_cost_limit + } +} + +#[cfg(test)] +mod tests { + use { + crate::bundle_stage::bundle_reserved_space_manager::BundleReservedSpaceManager, + solana_ledger::genesis_utils::create_genesis_config, solana_runtime::bank::Bank, + solana_sdk::pubkey::Pubkey, std::sync::Arc, + }; + + #[test] + fn test_reserve_block_cost_limits_during_reserved_ticks() { + const BUNDLE_BLOCK_COST_LIMITS_RESERVATION: u64 = 100; + + let genesis_config_info = create_genesis_config(100); + let bank = Arc::new(Bank::new_for_tests(&genesis_config_info.genesis_config)); + + let block_cost_limits = bank.read_cost_tracker().unwrap().block_cost_limit(); + + let mut reserved_space = BundleReservedSpaceManager::new( + block_cost_limits, + BUNDLE_BLOCK_COST_LIMITS_RESERVATION, + 5, + ); + reserved_space.tick(&bank); + + assert_eq!( + bank.read_cost_tracker().unwrap().block_cost_limit(), + block_cost_limits - BUNDLE_BLOCK_COST_LIMITS_RESERVATION + ); + } + + #[test] + fn test_dont_reserve_block_cost_limits_after_reserved_ticks() { + const BUNDLE_BLOCK_COST_LIMITS_RESERVATION: u64 = 100; + + let genesis_config_info = create_genesis_config(100); + let bank = Arc::new(Bank::new_for_tests(&genesis_config_info.genesis_config)); + + let block_cost_limits = bank.read_cost_tracker().unwrap().block_cost_limit(); + + for _ in 0..5 { + bank.register_default_tick_for_test(); + } + + let mut reserved_space = BundleReservedSpaceManager::new( + block_cost_limits, + BUNDLE_BLOCK_COST_LIMITS_RESERVATION, + 5, + ); + reserved_space.tick(&bank); + + assert_eq!( + bank.read_cost_tracker().unwrap().block_cost_limit(), + block_cost_limits + ); + } + + #[test] + fn test_dont_reset_block_cost_limits_during_reserved_ticks() { + const BUNDLE_BLOCK_COST_LIMITS_RESERVATION: u64 = 100; + + let genesis_config_info = create_genesis_config(100); + let bank = Arc::new(Bank::new_for_tests(&genesis_config_info.genesis_config)); + + let block_cost_limits = bank.read_cost_tracker().unwrap().block_cost_limit(); + + let mut reserved_space = BundleReservedSpaceManager::new( + block_cost_limits, + BUNDLE_BLOCK_COST_LIMITS_RESERVATION, + 5, + ); + + reserved_space.tick(&bank); + bank.register_default_tick_for_test(); + reserved_space.tick(&bank); + + assert_eq!( + bank.read_cost_tracker().unwrap().block_cost_limit(), + block_cost_limits - BUNDLE_BLOCK_COST_LIMITS_RESERVATION + ); + } + + #[test] + fn test_reset_block_cost_limits_after_reserved_ticks() { + const BUNDLE_BLOCK_COST_LIMITS_RESERVATION: u64 = 100; + + let genesis_config_info = create_genesis_config(100); + let bank = Arc::new(Bank::new_for_tests(&genesis_config_info.genesis_config)); + + let block_cost_limits = bank.read_cost_tracker().unwrap().block_cost_limit(); + + let mut reserved_space = BundleReservedSpaceManager::new( + block_cost_limits, + BUNDLE_BLOCK_COST_LIMITS_RESERVATION, + 5, + ); + + reserved_space.tick(&bank); + + for _ in 0..5 { + bank.register_default_tick_for_test(); + } + reserved_space.tick(&bank); + + assert_eq!( + bank.read_cost_tracker().unwrap().block_cost_limit(), + block_cost_limits + ); + } + + #[test] + fn test_block_limits_after_first_slot() { + const BUNDLE_BLOCK_COST_LIMITS_RESERVATION: u64 = 100; + const RESERVED_TICKS: u64 = 5; + let genesis_config_info = create_genesis_config(100); + let bank = Arc::new(Bank::new_for_tests(&genesis_config_info.genesis_config)); + + for _ in 0..genesis_config_info.genesis_config.ticks_per_slot { + bank.register_default_tick_for_test(); + } + assert!(bank.is_complete()); + bank.freeze(); + assert_eq!( + bank.read_cost_tracker().unwrap().block_cost_limit(), + solana_cost_model::block_cost_limits::MAX_BLOCK_UNITS, + ); + + let bank1 = Arc::new(Bank::new_from_parent(bank.clone(), &Pubkey::default(), 1)); + assert_eq!(bank1.slot(), 1); + assert_eq!(bank1.tick_height(), 64); + assert_eq!(bank1.max_tick_height(), 128); + + // reserve space + let block_cost_limits = bank1.read_cost_tracker().unwrap().block_cost_limit(); + let mut reserved_space = BundleReservedSpaceManager::new( + block_cost_limits, + BUNDLE_BLOCK_COST_LIMITS_RESERVATION, + RESERVED_TICKS, + ); + reserved_space.tick(&bank1); + + // wait for reservation to be over + (0..RESERVED_TICKS).for_each(|_| { + bank1.register_default_tick_for_test(); + assert_eq!( + bank1.read_cost_tracker().unwrap().block_cost_limit(), + block_cost_limits - BUNDLE_BLOCK_COST_LIMITS_RESERVATION + ); + }); + reserved_space.tick(&bank1); + + // after reservation, revert back to normal limit + assert_eq!( + bank1.read_cost_tracker().unwrap().block_cost_limit(), + solana_cost_model::block_cost_limits::MAX_BLOCK_UNITS, + ); + } +} diff --git a/core/src/bundle_stage/bundle_stage_leader_metrics.rs b/core/src/bundle_stage/bundle_stage_leader_metrics.rs new file mode 100644 index 0000000000..97bbe7ac29 --- /dev/null +++ b/core/src/bundle_stage/bundle_stage_leader_metrics.rs @@ -0,0 +1,506 @@ +use { + crate::{ + banking_stage::{ + leader_slot_metrics::{self, LeaderSlotMetricsTracker}, + unprocessed_transaction_storage::UnprocessedTransactionStorage, + }, + immutable_deserialized_bundle::DeserializedBundleError, + }, + solana_bundle::{bundle_execution::LoadAndExecuteBundleError, BundleExecutionError}, + solana_poh::poh_recorder::BankStart, + solana_sdk::{bundle::SanitizedBundle, clock::Slot, saturating_add_assign}, +}; + +pub struct BundleStageLeaderMetrics { + bundle_stage_metrics_tracker: BundleStageStatsMetricsTracker, + leader_slot_metrics_tracker: LeaderSlotMetricsTracker, +} + +pub(crate) enum MetricsTrackerAction { + Noop, + ReportAndResetTracker, + NewTracker(Option), + ReportAndNewTracker(Option), +} + +impl BundleStageLeaderMetrics { + pub fn new(id: u32) -> Self { + Self { + bundle_stage_metrics_tracker: BundleStageStatsMetricsTracker::new(id), + leader_slot_metrics_tracker: LeaderSlotMetricsTracker::new(id), + } + } + + pub(crate) fn check_leader_slot_boundary( + &mut self, + bank_start: Option<&BankStart>, + unprocessed_transaction_storage: Option<&UnprocessedTransactionStorage>, + ) -> ( + leader_slot_metrics::MetricsTrackerAction, + MetricsTrackerAction, + ) { + let banking_stage_metrics_action = self + .leader_slot_metrics_tracker + .check_leader_slot_boundary(bank_start, unprocessed_transaction_storage); + let bundle_stage_metrics_action = self + .bundle_stage_metrics_tracker + .check_leader_slot_boundary(bank_start); + (banking_stage_metrics_action, bundle_stage_metrics_action) + } + + pub(crate) fn apply_action( + &mut self, + banking_stage_metrics_action: leader_slot_metrics::MetricsTrackerAction, + bundle_stage_metrics_action: MetricsTrackerAction, + ) -> Option { + self.leader_slot_metrics_tracker + .apply_action(banking_stage_metrics_action); + self.bundle_stage_metrics_tracker + .apply_action(bundle_stage_metrics_action) + } + + pub fn leader_slot_metrics_tracker(&mut self) -> &mut LeaderSlotMetricsTracker { + &mut self.leader_slot_metrics_tracker + } + + pub fn bundle_stage_metrics_tracker(&mut self) -> &mut BundleStageStatsMetricsTracker { + &mut self.bundle_stage_metrics_tracker + } +} + +pub struct BundleStageStatsMetricsTracker { + bundle_stage_metrics: Option, + id: u32, +} + +impl BundleStageStatsMetricsTracker { + pub fn new(id: u32) -> Self { + Self { + bundle_stage_metrics: None, + id, + } + } + + /// Similar to as LeaderSlotMetricsTracker::check_leader_slot_boundary + pub(crate) fn check_leader_slot_boundary( + &mut self, + bank_start: Option<&BankStart>, + ) -> MetricsTrackerAction { + match (self.bundle_stage_metrics.as_mut(), bank_start) { + (None, None) => MetricsTrackerAction::Noop, + (Some(_), None) => MetricsTrackerAction::ReportAndResetTracker, + // Our leader slot has begun, time to create a new slot tracker + (None, Some(bank_start)) => MetricsTrackerAction::NewTracker(Some( + BundleStageStats::new(self.id, bank_start.working_bank.slot()), + )), + (Some(bundle_stage_metrics), Some(bank_start)) => { + if bundle_stage_metrics.slot != bank_start.working_bank.slot() { + // Last slot has ended, new slot has began + MetricsTrackerAction::ReportAndNewTracker(Some(BundleStageStats::new( + self.id, + bank_start.working_bank.slot(), + ))) + } else { + MetricsTrackerAction::Noop + } + } + } + } + + /// Similar to LeaderSlotMetricsTracker::apply_action + pub(crate) fn apply_action(&mut self, action: MetricsTrackerAction) -> Option { + match action { + MetricsTrackerAction::Noop => None, + MetricsTrackerAction::ReportAndResetTracker => { + let mut reported_slot = None; + if let Some(bundle_stage_metrics) = self.bundle_stage_metrics.as_mut() { + bundle_stage_metrics.report(); + reported_slot = bundle_stage_metrics.reported_slot(); + } + self.bundle_stage_metrics = None; + reported_slot + } + MetricsTrackerAction::NewTracker(new_bundle_stage_metrics) => { + self.bundle_stage_metrics = new_bundle_stage_metrics; + self.bundle_stage_metrics.as_ref().unwrap().reported_slot() + } + MetricsTrackerAction::ReportAndNewTracker(new_bundle_stage_metrics) => { + let mut reported_slot = None; + if let Some(bundle_stage_metrics) = self.bundle_stage_metrics.as_mut() { + bundle_stage_metrics.report(); + reported_slot = bundle_stage_metrics.reported_slot(); + } + self.bundle_stage_metrics = new_bundle_stage_metrics; + reported_slot + } + } + } + + pub(crate) fn increment_sanitize_transaction_result( + &mut self, + result: &Result, + ) { + if let Some(bundle_stage_metrics) = self.bundle_stage_metrics.as_mut() { + match result { + Ok(_) => { + saturating_add_assign!(bundle_stage_metrics.sanitize_transaction_ok, 1); + } + Err(e) => match e { + DeserializedBundleError::VoteOnlyMode => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_vote_only_mode, + 1 + ); + } + DeserializedBundleError::BlacklistedAccount => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_blacklisted_account, + 1 + ); + } + DeserializedBundleError::FailedToSerializeTransaction => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_failed_to_serialize, + 1 + ); + } + DeserializedBundleError::DuplicateTransaction => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_duplicate_transaction, + 1 + ); + } + DeserializedBundleError::FailedCheckTransactions => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_failed_check, + 1 + ); + } + DeserializedBundleError::FailedToSerializePacket => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_failed_to_serialize, + 1 + ); + } + DeserializedBundleError::EmptyBatch => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_failed_empty_batch, + 1 + ); + } + DeserializedBundleError::TooManyPackets => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_failed_too_many_packets, + 1 + ); + } + DeserializedBundleError::MarkedDiscard => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_failed_marked_discard, + 1 + ); + } + DeserializedBundleError::SignatureVerificationFailure => { + saturating_add_assign!( + bundle_stage_metrics.sanitize_transaction_failed_sig_verify_failed, + 1 + ); + } + }, + } + } + } + + pub fn increment_bundle_execution_result(&mut self, result: &Result<(), BundleExecutionError>) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + match result { + Ok(_) => { + saturating_add_assign!(bundle_stage_metrics.execution_results_ok, 1); + } + Err(BundleExecutionError::PohRecordError(_)) + | Err(BundleExecutionError::BankProcessingTimeLimitReached) => { + saturating_add_assign!( + bundle_stage_metrics.execution_results_poh_max_height, + 1 + ); + } + Err(BundleExecutionError::TransactionFailure( + LoadAndExecuteBundleError::ProcessingTimeExceeded(_), + )) => { + saturating_add_assign!(bundle_stage_metrics.num_execution_timeouts, 1); + } + Err(BundleExecutionError::TransactionFailure( + LoadAndExecuteBundleError::TransactionError { .. }, + )) => { + saturating_add_assign!( + bundle_stage_metrics.execution_results_transaction_failures, + 1 + ); + } + Err(BundleExecutionError::TransactionFailure( + LoadAndExecuteBundleError::LockError { .. }, + )) + | Err(BundleExecutionError::LockError) => { + saturating_add_assign!(bundle_stage_metrics.num_lock_errors, 1); + } + Err(BundleExecutionError::ExceedsCostModel) => { + saturating_add_assign!( + bundle_stage_metrics.execution_results_exceeds_cost_model, + 1 + ); + } + Err(BundleExecutionError::TipError(_)) => { + saturating_add_assign!(bundle_stage_metrics.execution_results_tip_errors, 1); + } + Err(BundleExecutionError::TransactionFailure( + LoadAndExecuteBundleError::InvalidPreOrPostAccounts, + )) => { + saturating_add_assign!(bundle_stage_metrics.bad_argument, 1); + } + } + } + } + + pub(crate) fn increment_sanitize_bundle_elapsed_us(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.sanitize_bundle_elapsed_us, count); + } + } + + pub(crate) fn increment_locked_bundle_elapsed_us(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.locked_bundle_elapsed_us, count); + } + } + + pub(crate) fn increment_num_init_tip_account_errors(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.num_init_tip_account_errors, count); + } + } + + pub(crate) fn increment_num_init_tip_account_ok(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.num_init_tip_account_ok, count); + } + } + + pub(crate) fn increment_num_change_tip_receiver_errors(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.num_change_tip_receiver_errors, count); + } + } + + pub(crate) fn increment_num_change_tip_receiver_ok(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.num_change_tip_receiver_ok, count); + } + } + + pub(crate) fn increment_change_tip_receiver_elapsed_us(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.change_tip_receiver_elapsed_us, count); + } + } + + pub(crate) fn increment_num_execution_retries(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!(bundle_stage_metrics.num_execution_retries, count); + } + } + + pub(crate) fn increment_execute_locked_bundles_elapsed_us(&mut self, count: u64) { + if let Some(bundle_stage_metrics) = &mut self.bundle_stage_metrics { + saturating_add_assign!( + bundle_stage_metrics.execute_locked_bundles_elapsed_us, + count + ); + } + } +} + +#[derive(Default)] +pub struct BundleStageStats { + id: u32, + slot: u64, + is_reported: bool, + + sanitize_transaction_ok: u64, + sanitize_transaction_vote_only_mode: u64, + sanitize_transaction_blacklisted_account: u64, + sanitize_transaction_failed_to_serialize: u64, + sanitize_transaction_duplicate_transaction: u64, + sanitize_transaction_failed_check: u64, + sanitize_bundle_elapsed_us: u64, + sanitize_transaction_failed_empty_batch: u64, + sanitize_transaction_failed_too_many_packets: u64, + sanitize_transaction_failed_marked_discard: u64, + sanitize_transaction_failed_sig_verify_failed: u64, + + locked_bundle_elapsed_us: u64, + + num_lock_errors: u64, + + num_init_tip_account_errors: u64, + num_init_tip_account_ok: u64, + + num_change_tip_receiver_errors: u64, + num_change_tip_receiver_ok: u64, + change_tip_receiver_elapsed_us: u64, + + num_execution_timeouts: u64, + num_execution_retries: u64, + + execute_locked_bundles_elapsed_us: u64, + + execution_results_ok: u64, + execution_results_poh_max_height: u64, + execution_results_transaction_failures: u64, + execution_results_exceeds_cost_model: u64, + execution_results_tip_errors: u64, + execution_results_max_retries: u64, + + bad_argument: u64, +} + +impl BundleStageStats { + pub fn new(id: u32, slot: Slot) -> BundleStageStats { + BundleStageStats { + id, + slot, + is_reported: false, + ..BundleStageStats::default() + } + } + + /// Returns `Some(self.slot)` if the metrics have been reported, otherwise returns None + fn reported_slot(&self) -> Option { + if self.is_reported { + Some(self.slot) + } else { + None + } + } + + pub fn report(&mut self) { + self.is_reported = true; + + datapoint_info!( + "bundle_stage-stats", + ("id", self.id, i64), + ("slot", self.slot, i64), + ("num_sanitized_ok", self.sanitize_transaction_ok, i64), + ( + "sanitize_transaction_vote_only_mode", + self.sanitize_transaction_vote_only_mode, + i64 + ), + ( + "sanitize_transaction_blacklisted_account", + self.sanitize_transaction_blacklisted_account, + i64 + ), + ( + "sanitize_transaction_failed_to_serialize", + self.sanitize_transaction_failed_to_serialize, + i64 + ), + ( + "sanitize_transaction_duplicate_transaction", + self.sanitize_transaction_duplicate_transaction, + i64 + ), + ( + "sanitize_transaction_failed_check", + self.sanitize_transaction_failed_check, + i64 + ), + ( + "sanitize_bundle_elapsed_us", + self.sanitize_bundle_elapsed_us, + i64 + ), + ( + "sanitize_transaction_failed_empty_batch", + self.sanitize_transaction_failed_empty_batch, + i64 + ), + ( + "sanitize_transaction_failed_too_many_packets", + self.sanitize_transaction_failed_too_many_packets, + i64 + ), + ( + "sanitize_transaction_failed_marked_discard", + self.sanitize_transaction_failed_marked_discard, + i64 + ), + ( + "sanitize_transaction_failed_sig_verify_failed", + self.sanitize_transaction_failed_sig_verify_failed, + i64 + ), + ( + "locked_bundle_elapsed_us", + self.locked_bundle_elapsed_us, + i64 + ), + ("num_lock_errors", self.num_lock_errors, i64), + ( + "num_init_tip_account_errors", + self.num_init_tip_account_errors, + i64 + ), + ("num_init_tip_account_ok", self.num_init_tip_account_ok, i64), + ( + "num_change_tip_receiver_errors", + self.num_change_tip_receiver_errors, + i64 + ), + ( + "num_change_tip_receiver_ok", + self.num_change_tip_receiver_ok, + i64 + ), + ( + "change_tip_receiver_elapsed_us", + self.change_tip_receiver_elapsed_us, + i64 + ), + ("num_execution_timeouts", self.num_execution_timeouts, i64), + ("num_execution_retries", self.num_execution_retries, i64), + ( + "execute_locked_bundles_elapsed_us", + self.execute_locked_bundles_elapsed_us, + i64 + ), + ("execution_results_ok", self.execution_results_ok, i64), + ( + "execution_results_poh_max_height", + self.execution_results_poh_max_height, + i64 + ), + ( + "execution_results_transaction_failures", + self.execution_results_transaction_failures, + i64 + ), + ( + "execution_results_exceeds_cost_model", + self.execution_results_exceeds_cost_model, + i64 + ), + ( + "execution_results_tip_errors", + self.execution_results_tip_errors, + i64 + ), + ( + "execution_results_max_retries", + self.execution_results_max_retries, + i64 + ), + ("bad_argument", self.bad_argument, i64) + ); + } +} diff --git a/core/src/bundle_stage/committer.rs b/core/src/bundle_stage/committer.rs new file mode 100644 index 0000000000..03db5694a4 --- /dev/null +++ b/core/src/bundle_stage/committer.rs @@ -0,0 +1,227 @@ +use { + crate::banking_stage::{ + committer::CommitTransactionDetails, + leader_slot_timing_metrics::LeaderExecuteAndCommitTimings, + }, + solana_bundle::bundle_execution::LoadAndExecuteBundleOutput, + solana_ledger::blockstore_processor::TransactionStatusSender, + solana_measure::measure_us, + solana_runtime::{ + bank::{Bank, ExecutedTransactionCounts, TransactionBalances, TransactionBalancesSet}, + bank_utils, + prioritization_fee_cache::PrioritizationFeeCache, + vote_sender_types::ReplayVoteSender, + }, + solana_sdk::{hash::Hash, saturating_add_assign, transaction::SanitizedTransaction}, + solana_svm::transaction_results::TransactionResults, + solana_transaction_status::{ + token_balances::{TransactionTokenBalances, TransactionTokenBalancesSet}, + PreBalanceInfo, + }, + std::sync::Arc, +}; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CommitBundleDetails { + pub commit_transaction_details: Vec>, +} + +pub struct Committer { + transaction_status_sender: Option, + replay_vote_sender: ReplayVoteSender, + prioritization_fee_cache: Arc, +} + +impl Committer { + pub fn new( + transaction_status_sender: Option, + replay_vote_sender: ReplayVoteSender, + prioritization_fee_cache: Arc, + ) -> Self { + Self { + transaction_status_sender, + replay_vote_sender, + prioritization_fee_cache, + } + } + + pub(crate) fn transaction_status_sender_enabled(&self) -> bool { + self.transaction_status_sender.is_some() + } + + /// Very similar to Committer::commit_transactions, but works with bundles. + /// The main difference is there's multiple non-parallelizable transaction vectors to commit + /// and post-balances are collected after execution instead of from the bank in Self::collect_balances_and_send_status_batch. + #[allow(clippy::too_many_arguments)] + pub(crate) fn commit_bundle<'a>( + &self, + bundle_execution_output: &'a mut LoadAndExecuteBundleOutput<'a>, + last_blockhash: Hash, + lamports_per_signature: u64, + mut starting_transaction_index: Option, + bank: &Arc, + execute_and_commit_timings: &mut LeaderExecuteAndCommitTimings, + ) -> (u64, CommitBundleDetails) { + let transaction_output = bundle_execution_output.bundle_transaction_results_mut(); + + let (commit_transaction_details, commit_times): (Vec<_>, Vec<_>) = transaction_output + .iter_mut() + .map(|bundle_results| { + let executed_transactions_count = bundle_results + .load_and_execute_transactions_output() + .executed_transactions_count + as u64; + + let executed_non_vote_transactions_count = bundle_results + .load_and_execute_transactions_output() + .executed_non_vote_transactions_count + as u64; + + let executed_with_failure_result_count = bundle_results + .load_and_execute_transactions_output() + .executed_transactions_count + .saturating_sub( + bundle_results + .load_and_execute_transactions_output() + .executed_with_successful_result_count, + ) as u64; + + let signature_count = bundle_results + .load_and_execute_transactions_output() + .signature_count; + + let sanitized_transactions = bundle_results.transactions().to_vec(); + let execution_results = bundle_results.execution_results().to_vec(); + + let loaded_transactions = bundle_results.loaded_transactions_mut(); + debug!("loaded_transactions: {:?}", loaded_transactions); + + let (tx_results, commit_time_us) = measure_us!(bank.commit_transactions( + &sanitized_transactions, + loaded_transactions, + execution_results, + last_blockhash, + lamports_per_signature, + ExecutedTransactionCounts { + executed_transactions_count, + executed_non_vote_transactions_count, + executed_with_failure_result_count, + signature_count, + }, + &mut execute_and_commit_timings.execute_timings, + )); + + let commit_transaction_statuses: Vec<_> = tx_results + .execution_results + .iter() + .zip(tx_results.loaded_accounts_stats.iter()) + .map(|(execution_result, loaded_accounts_stats)| { + match execution_result.details() { + // reports actual execution CUs, and actual loaded accounts size for + // transaction committed to block. qos_service uses these information to adjust + // reserved block space. + Some(details) => CommitTransactionDetails::Committed { + compute_units: details.executed_units, + loaded_accounts_data_size: loaded_accounts_stats + .as_ref() + .map_or(0, |stats| stats.loaded_accounts_data_size), + }, + None => CommitTransactionDetails::NotCommitted, + } + }) + .collect(); + + let ((), find_and_send_votes_us) = measure_us!({ + bank_utils::find_and_send_votes( + &sanitized_transactions, + &tx_results, + Some(&self.replay_vote_sender), + ); + + let post_balance_info = bundle_results.post_balance_info().clone(); + let pre_balance_info = bundle_results.pre_balance_info(); + + let num_committed = tx_results + .execution_results + .iter() + .filter(|r| r.was_executed()) + .count(); + + self.collect_balances_and_send_status_batch( + tx_results, + bank, + sanitized_transactions, + pre_balance_info, + post_balance_info, + starting_transaction_index, + ); + + // NOTE: we're doing batched records, so we need to increment the poh starting_transaction_index + // by number committed so the next batch will have the correct starting_transaction_index + starting_transaction_index = + starting_transaction_index.map(|starting_transaction_index| { + starting_transaction_index.saturating_add(num_committed) + }); + + self.prioritization_fee_cache + .update(bank, bundle_results.executed_transactions().into_iter()); + }); + saturating_add_assign!( + execute_and_commit_timings.find_and_send_votes_us, + find_and_send_votes_us + ); + + (commit_transaction_statuses, commit_time_us) + }) + .unzip(); + + ( + commit_times.iter().sum(), + CommitBundleDetails { + commit_transaction_details, + }, + ) + } + + fn collect_balances_and_send_status_batch( + &self, + tx_results: TransactionResults, + bank: &Arc, + sanitized_transactions: Vec, + pre_balance_info: &mut PreBalanceInfo, + (post_balances, post_token_balances): (TransactionBalances, TransactionTokenBalances), + starting_transaction_index: Option, + ) { + if let Some(transaction_status_sender) = &self.transaction_status_sender { + let mut transaction_index = starting_transaction_index.unwrap_or_default(); + let batch_transaction_indexes: Vec<_> = tx_results + .execution_results + .iter() + .map(|result| { + if result.was_executed() { + let this_transaction_index = transaction_index; + saturating_add_assign!(transaction_index, 1); + this_transaction_index + } else { + 0 + } + }) + .collect(); + transaction_status_sender.send_transaction_status_batch( + bank.clone(), + sanitized_transactions, + tx_results.execution_results, + TransactionBalancesSet::new( + std::mem::take(&mut pre_balance_info.native), + post_balances, + ), + TransactionTokenBalancesSet::new( + std::mem::take(&mut pre_balance_info.token), + post_token_balances, + ), + tx_results.rent_debits, + batch_transaction_indexes, + ); + } + } +} diff --git a/core/src/bundle_stage/result.rs b/core/src/bundle_stage/result.rs new file mode 100644 index 0000000000..3370251791 --- /dev/null +++ b/core/src/bundle_stage/result.rs @@ -0,0 +1,41 @@ +use { + crate::{ + bundle_stage::bundle_account_locker::BundleAccountLockerError, tip_manager::TipPaymentError, + }, + anchor_lang::error::Error, + solana_bundle::bundle_execution::LoadAndExecuteBundleError, + solana_poh::poh_recorder::PohRecorderError, + thiserror::Error, +}; + +pub type BundleExecutionResult = Result; + +#[derive(Error, Debug, Clone)] +pub enum BundleExecutionError { + #[error("PoH record error: {0}")] + PohRecordError(#[from] PohRecorderError), + + #[error("Bank is done processing")] + BankProcessingDone, + + #[error("Execution error: {0}")] + ExecutionError(#[from] LoadAndExecuteBundleError), + + #[error("The bundle exceeds the cost model")] + ExceedsCostModel, + + #[error("Tip error {0}")] + TipError(#[from] TipPaymentError), + + #[error("Error locking bundle")] + LockError(#[from] BundleAccountLockerError), +} + +impl From for TipPaymentError { + fn from(anchor_err: Error) -> Self { + match anchor_err { + Error::AnchorError(e) => Self::AnchorError(e.error_msg), + Error::ProgramError(e) => Self::AnchorError(e.to_string()), + } + } +} diff --git a/core/src/consensus_cache_updater.rs b/core/src/consensus_cache_updater.rs new file mode 100644 index 0000000000..e1dc137ba0 --- /dev/null +++ b/core/src/consensus_cache_updater.rs @@ -0,0 +1,52 @@ +use { + solana_runtime::bank::Bank, + solana_sdk::{clock::Epoch, pubkey::Pubkey}, + std::collections::HashSet, +}; + +#[derive(Default)] +pub(crate) struct ConsensusCacheUpdater { + last_epoch_updated: Epoch, + consensus_accounts_cache: HashSet, +} + +impl ConsensusCacheUpdater { + pub(crate) fn consensus_accounts_cache(&self) -> &HashSet { + &self.consensus_accounts_cache + } + + /// Builds a HashSet of all consensus related accounts for the Bank's epoch + fn get_consensus_accounts(bank: &Bank) -> HashSet { + let mut consensus_accounts: HashSet = HashSet::new(); + if let Some(epoch_stakes) = bank.epoch_stakes(bank.epoch()) { + // votes use the following accounts: + // - vote_account pubkey: writeable + // - authorized_voter_pubkey: read-only + // - node_keypair pubkey: payer (writeable) + let node_id_vote_accounts = epoch_stakes.node_id_to_vote_accounts(); + + let vote_accounts = node_id_vote_accounts + .values() + .flat_map(|v| v.vote_accounts.clone()); + + // vote_account + consensus_accounts.extend(vote_accounts); + // authorized_voter_pubkey + consensus_accounts.extend(epoch_stakes.epoch_authorized_voters().keys()); + // node_keypair + consensus_accounts.extend(epoch_stakes.node_id_to_vote_accounts().keys()); + } + consensus_accounts + } + + /// Updates consensus-related accounts on epoch boundaries + pub(crate) fn maybe_update(&mut self, bank: &Bank) -> bool { + if bank.epoch() > self.last_epoch_updated { + self.consensus_accounts_cache = Self::get_consensus_accounts(bank); + self.last_epoch_updated = bank.epoch(); + true + } else { + false + } + } +} diff --git a/core/src/immutable_deserialized_bundle.rs b/core/src/immutable_deserialized_bundle.rs new file mode 100644 index 0000000000..aa1c395f24 --- /dev/null +++ b/core/src/immutable_deserialized_bundle.rs @@ -0,0 +1,488 @@ +use { + crate::{ + banking_stage::immutable_deserialized_packet::ImmutableDeserializedPacket, + packet_bundle::PacketBundle, + }, + solana_perf::sigverify::verify_packet, + solana_runtime::bank::Bank, + solana_sdk::{ + bundle::SanitizedBundle, clock::MAX_PROCESSING_AGE, pubkey::Pubkey, signature::Signature, + transaction::SanitizedTransaction, + }, + solana_svm::transaction_error_metrics::TransactionErrorMetrics, + std::{ + collections::{hash_map::RandomState, HashSet}, + iter::repeat, + }, + thiserror::Error, +}; + +#[derive(Debug, Error, Eq, PartialEq)] +pub enum DeserializedBundleError { + #[error("FailedToSerializePacket")] + FailedToSerializePacket, + + #[error("EmptyBatch")] + EmptyBatch, + + #[error("TooManyPackets")] + TooManyPackets, + + #[error("MarkedDiscard")] + MarkedDiscard, + + #[error("SignatureVerificationFailure")] + SignatureVerificationFailure, + + #[error("Bank is in vote-only mode")] + VoteOnlyMode, + + #[error("Bundle mentions blacklisted account")] + BlacklistedAccount, + + #[error("Bundle contains a transaction that failed to serialize")] + FailedToSerializeTransaction, + + #[error("Bundle contains a duplicate transaction")] + DuplicateTransaction, + + #[error("Bundle failed check_transactions")] + FailedCheckTransactions, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ImmutableDeserializedBundle { + bundle_id: String, + packets: Vec, +} + +impl ImmutableDeserializedBundle { + pub fn new( + bundle: &mut PacketBundle, + max_len: Option, + ) -> Result { + // Checks: non-zero, less than some length, marked for discard, signature verification failed, failed to sanitize to + // ImmutableDeserializedPacket + if bundle.batch.is_empty() { + return Err(DeserializedBundleError::EmptyBatch); + } + if max_len + .map(|max_len| bundle.batch.len() > max_len) + .unwrap_or(false) + { + return Err(DeserializedBundleError::TooManyPackets); + } + if bundle.batch.iter().any(|p| p.meta().discard()) { + return Err(DeserializedBundleError::MarkedDiscard); + } + if bundle.batch.iter_mut().any(|p| !verify_packet(p, false)) { + return Err(DeserializedBundleError::SignatureVerificationFailure); + } + + let immutable_packets: Vec<_> = bundle + .batch + .iter() + .filter_map(|p| ImmutableDeserializedPacket::new(p.clone()).ok()) + .collect(); + + if bundle.batch.len() != immutable_packets.len() { + return Err(DeserializedBundleError::FailedToSerializePacket); + } + + Ok(Self { + bundle_id: bundle.bundle_id.clone(), + packets: immutable_packets, + }) + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.packets.len() + } + + pub fn bundle_id(&self) -> &str { + &self.bundle_id + } + + /// A bundle has the following requirements: + /// - all transactions must be sanitiz-able + /// - no duplicate signatures + /// - must not contain a blacklisted account + /// - can't already be processed or contain a bad blockhash + pub fn build_sanitized_bundle( + &self, + bank: &Bank, + blacklisted_accounts: &HashSet, + transaction_error_metrics: &mut TransactionErrorMetrics, + ) -> Result { + if bank.vote_only_bank() { + return Err(DeserializedBundleError::VoteOnlyMode); + } + + let transactions: Vec = self + .packets + .iter() + .filter_map(|p| { + p.build_sanitized_transaction( + bank.vote_only_bank(), + bank, + bank.get_reserved_account_keys(), + ) + }) + .collect(); + + if self.packets.len() != transactions.len() { + return Err(DeserializedBundleError::FailedToSerializeTransaction); + } + + let unique_signatures: HashSet<&Signature, RandomState> = + HashSet::from_iter(transactions.iter().map(|tx| tx.signature())); + if unique_signatures.len() != transactions.len() { + return Err(DeserializedBundleError::DuplicateTransaction); + } + + let contains_blacklisted_account = transactions.iter().any(|tx| { + tx.message() + .account_keys() + .iter() + .any(|acc| blacklisted_accounts.contains(acc)) + }); + + if contains_blacklisted_account { + return Err(DeserializedBundleError::BlacklistedAccount); + } + + // assume everything locks okay to check for already-processed transaction or expired/invalid blockhash + let lock_results: Vec<_> = repeat(Ok(())).take(transactions.len()).collect(); + let check_results = bank.check_transactions( + &transactions, + &lock_results, + MAX_PROCESSING_AGE, + transaction_error_metrics, + ); + + if check_results.iter().any(|r| r.is_err()) { + return Err(DeserializedBundleError::FailedCheckTransactions); + } + + Ok(SanitizedBundle { + transactions, + bundle_id: self.bundle_id.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use { + crate::{ + immutable_deserialized_bundle::{DeserializedBundleError, ImmutableDeserializedBundle}, + packet_bundle::PacketBundle, + }, + solana_client::rpc_client::SerializableTransaction, + solana_ledger::genesis_utils::create_genesis_config, + solana_perf::packet::PacketBatch, + solana_runtime::{ + bank::{Bank, NewBankOptions}, + genesis_utils::GenesisConfigInfo, + }, + solana_sdk::{ + hash::Hash, + packet::Packet, + pubkey::Pubkey, + signature::{Keypair, Signer}, + system_transaction::transfer, + }, + solana_svm::transaction_error_metrics::TransactionErrorMetrics, + std::{collections::HashSet, sync::Arc}, + }; + /// Happy case + #[test] + fn test_simple_get_sanitized_bundle() { + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (bank, _) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let kp = Keypair::new(); + + let tx0 = transfer(&mint_keypair, &kp.pubkey(), 500, genesis_config.hash()); + + let tx1 = transfer(&mint_keypair, &kp.pubkey(), 501, genesis_config.hash()); + + let bundle = ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![ + Packet::from_data(None, &tx0).unwrap(), + Packet::from_data(None, &tx1).unwrap(), + ]), + bundle_id: String::default(), + }, + None, + ) + .unwrap(); + + let mut transaction_errors = TransactionErrorMetrics::default(); + let sanitized_bundle = bundle + .build_sanitized_bundle(&bank, &HashSet::default(), &mut transaction_errors) + .unwrap(); + assert_eq!(sanitized_bundle.transactions.len(), 2); + assert_eq!( + sanitized_bundle.transactions[0].signature(), + tx0.get_signature() + ); + assert_eq!( + sanitized_bundle.transactions[1].signature(), + tx1.get_signature() + ); + } + + #[test] + fn test_empty_batch_fails_to_init() { + assert_eq!( + ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![]), + bundle_id: String::default(), + }, + None, + ), + Err(DeserializedBundleError::EmptyBatch) + ); + } + + #[test] + fn test_too_many_packets_fails_to_init() { + let kp = Keypair::new(); + + assert_eq!( + ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new( + (0..10) + .map(|i| { + Packet::from_data( + None, + transfer(&kp, &kp.pubkey(), i, Hash::default()), + ) + .unwrap() + }) + .collect() + ), + bundle_id: String::default(), + }, + Some(5), + ), + Err(DeserializedBundleError::TooManyPackets) + ); + } + + #[test] + fn test_packets_marked_discard_fails_to_init() { + let kp = Keypair::new(); + + let mut packet = + Packet::from_data(None, transfer(&kp, &kp.pubkey(), 100, Hash::default())).unwrap(); + packet.meta_mut().set_discard(true); + + assert_eq!( + ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![packet]), + bundle_id: String::default(), + }, + Some(5), + ), + Err(DeserializedBundleError::MarkedDiscard) + ); + } + + #[test] + fn test_bad_signature_fails_to_init() { + let kp0 = Keypair::new(); + let kp1 = Keypair::new(); + + let mut tx0 = transfer(&kp0, &kp0.pubkey(), 100, Hash::default()); + let tx1 = transfer(&kp1, &kp0.pubkey(), 100, Hash::default()); + tx0.signatures = tx1.signatures; + + assert_eq!( + ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data(None, tx0).unwrap()]), + bundle_id: String::default(), + }, + None + ), + Err(DeserializedBundleError::SignatureVerificationFailure) + ); + } + + #[test] + fn test_vote_only_bank_fails_to_build() { + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (parent, _) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + let vote_only_bank = Arc::new(Bank::new_from_parent_with_options( + parent, + &Pubkey::new_unique(), + 1, + NewBankOptions { + vote_only_bank: true, + }, + )); + + let kp = Keypair::new(); + + let tx0 = transfer(&mint_keypair, &kp.pubkey(), 500, genesis_config.hash()); + + let bundle = ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data(None, tx0).unwrap()]), + bundle_id: String::default(), + }, + None, + ) + .unwrap(); + + let mut transaction_errors = TransactionErrorMetrics::default(); + assert_matches!( + bundle.build_sanitized_bundle( + &vote_only_bank, + &HashSet::default(), + &mut transaction_errors + ), + Err(DeserializedBundleError::VoteOnlyMode) + ); + } + + #[test] + fn test_duplicate_signature_fails_to_build() { + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (bank, _) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let kp = Keypair::new(); + + let tx0 = transfer(&mint_keypair, &kp.pubkey(), 500, genesis_config.hash()); + + let bundle = ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![ + Packet::from_data(None, &tx0).unwrap(), + Packet::from_data(None, &tx0).unwrap(), + ]), + bundle_id: String::default(), + }, + None, + ) + .unwrap(); + + let mut transaction_errors = TransactionErrorMetrics::default(); + assert_matches!( + bundle.build_sanitized_bundle(&bank, &HashSet::default(), &mut transaction_errors), + Err(DeserializedBundleError::DuplicateTransaction) + ); + } + + #[test] + fn test_blacklisted_account_fails_to_build() { + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (bank, _) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let kp = Keypair::new(); + + let tx0 = transfer(&mint_keypair, &kp.pubkey(), 500, genesis_config.hash()); + + let bundle = ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data(None, tx0).unwrap()]), + bundle_id: String::default(), + }, + None, + ) + .unwrap(); + + let mut transaction_errors = TransactionErrorMetrics::default(); + assert_matches!( + bundle.build_sanitized_bundle( + &bank, + &HashSet::from([kp.pubkey()]), + &mut transaction_errors + ), + Err(DeserializedBundleError::BlacklistedAccount) + ); + } + + #[test] + fn test_already_processed_tx_fails_to_build() { + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (bank, _bank_forks) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let kp = Keypair::new(); + + let tx0 = transfer(&mint_keypair, &kp.pubkey(), 500, genesis_config.hash()); + + bank.process_transaction(&tx0).unwrap(); + + let bundle = ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data(None, tx0).unwrap()]), + bundle_id: String::default(), + }, + None, + ) + .unwrap(); + + let mut transaction_errors = TransactionErrorMetrics::default(); + assert_matches!( + bundle.build_sanitized_bundle(&bank, &HashSet::default(), &mut transaction_errors), + Err(DeserializedBundleError::FailedCheckTransactions) + ); + } + + #[test] + fn test_bad_blockhash_fails_to_build() { + let GenesisConfigInfo { + genesis_config, + mint_keypair, + .. + } = create_genesis_config(10_000); + let (bank, _) = Bank::new_no_wallclock_throttle_for_tests(&genesis_config); + + let kp = Keypair::new(); + + let tx0 = transfer(&mint_keypair, &kp.pubkey(), 500, Hash::default()); + + let bundle = ImmutableDeserializedBundle::new( + &mut PacketBundle { + batch: PacketBatch::new(vec![Packet::from_data(None, tx0).unwrap()]), + bundle_id: String::default(), + }, + None, + ) + .unwrap(); + + let mut transaction_errors = TransactionErrorMetrics::default(); + assert_matches!( + bundle.build_sanitized_bundle(&bank, &HashSet::default(), &mut transaction_errors), + Err(DeserializedBundleError::FailedCheckTransactions) + ); + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index c6ab7b7e9c..5c4aa86523 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -12,20 +12,25 @@ pub mod accounts_hash_verifier; pub mod admin_rpc_post_init; pub mod banking_stage; pub mod banking_trace; +pub mod bundle_stage; pub mod cache_block_meta_service; pub mod cluster_info_vote_listener; pub mod cluster_slots_service; pub mod commitment_service; pub mod completed_data_sets_service; pub mod consensus; +pub mod consensus_cache_updater; pub mod cost_update_service; pub mod drop_bank_service; pub mod fetch_stage; pub mod gen_keys; +pub mod immutable_deserialized_bundle; pub mod next_leader; pub mod optimistic_confirmation_verifier; +pub mod packet_bundle; pub mod poh_timing_report_service; pub mod poh_timing_reporter; +pub mod proxy; pub mod repair; pub mod replay_stage; mod result; @@ -38,6 +43,7 @@ pub mod snapshot_packager_service; pub mod staked_nodes_updater_service; pub mod stats_reporter_service; pub mod system_monitor_service; +pub mod tip_manager; pub mod tpu; mod tpu_entry_notifier; pub mod tracer_packet_stats; @@ -66,3 +72,41 @@ extern crate solana_frozen_abi_macro; #[cfg(test)] #[macro_use] extern crate assert_matches; + +use { + solana_sdk::packet::{Meta, Packet, PacketFlags, PACKET_DATA_SIZE}, + std::{ + cmp::min, + net::{IpAddr, Ipv4Addr}, + }, +}; + +const UNKNOWN_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); + +// NOTE: last profiled at around 180ns +pub fn proto_packet_to_packet(p: jito_protos::proto::packet::Packet) -> Packet { + let mut data = [0; PACKET_DATA_SIZE]; + let copy_len = min(data.len(), p.data.len()); + data[..copy_len].copy_from_slice(&p.data[..copy_len]); + let mut packet = Packet::new(data, Meta::default()); + if let Some(meta) = p.meta { + packet.meta_mut().size = meta.size as usize; + packet.meta_mut().addr = meta.addr.parse().unwrap_or(UNKNOWN_IP); + packet.meta_mut().port = meta.port as u16; + if let Some(flags) = meta.flags { + if flags.simple_vote_tx { + packet.meta_mut().flags.insert(PacketFlags::SIMPLE_VOTE_TX); + } + if flags.forwarded { + packet.meta_mut().flags.insert(PacketFlags::FORWARDED); + } + if flags.tracer_packet { + packet.meta_mut().flags.insert(PacketFlags::TRACER_PACKET); + } + if flags.repair { + packet.meta_mut().flags.insert(PacketFlags::REPAIR); + } + } + } + packet +} diff --git a/core/src/packet_bundle.rs b/core/src/packet_bundle.rs new file mode 100644 index 0000000000..2158f37414 --- /dev/null +++ b/core/src/packet_bundle.rs @@ -0,0 +1,7 @@ +use solana_perf::packet::PacketBatch; + +#[derive(Clone, Debug)] +pub struct PacketBundle { + pub batch: PacketBatch, + pub bundle_id: String, +} diff --git a/core/src/proxy/auth.rs b/core/src/proxy/auth.rs new file mode 100644 index 0000000000..39821e12ef --- /dev/null +++ b/core/src/proxy/auth.rs @@ -0,0 +1,185 @@ +use { + crate::proxy::ProxyError, + chrono::Utc, + jito_protos::proto::auth::{ + auth_service_client::AuthServiceClient, GenerateAuthChallengeRequest, + GenerateAuthTokensRequest, RefreshAccessTokenRequest, Role, Token, + }, + solana_gossip::cluster_info::ClusterInfo, + solana_sdk::signature::{Keypair, Signer}, + std::{ + sync::{Arc, Mutex}, + time::Duration, + }, + tokio::time::timeout, + tonic::{service::Interceptor, transport::Channel, Code, Request, Status}, +}; + +/// Interceptor responsible for adding the access token to request headers. +pub(crate) struct AuthInterceptor { + /// The token added to each request header. + access_token: Arc>, +} + +impl AuthInterceptor { + pub(crate) fn new(access_token: Arc>) -> Self { + Self { access_token } + } +} + +impl Interceptor for AuthInterceptor { + fn call(&mut self, mut request: Request<()>) -> Result, Status> { + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", self.access_token.lock().unwrap().value) + .parse() + .unwrap(), + ); + + Ok(request) + } +} + +/// Generates an auth challenge then generates and returns validated auth tokens. +pub async fn generate_auth_tokens( + auth_service_client: &mut AuthServiceClient, + // used to sign challenges + keypair: &Keypair, +) -> crate::proxy::Result<( + Token, /* access_token */ + Token, /* refresh_token */ +)> { + debug!("generate_auth_challenge"); + let challenge_response = auth_service_client + .generate_auth_challenge(GenerateAuthChallengeRequest { + role: Role::Validator as i32, + pubkey: keypair.pubkey().as_ref().to_vec(), + }) + .await + .map_err(|e: Status| { + if e.code() == Code::PermissionDenied { + ProxyError::AuthenticationPermissionDenied + } else { + ProxyError::AuthenticationError(e.to_string()) + } + })?; + + let formatted_challenge = format!( + "{}-{}", + keypair.pubkey(), + challenge_response.into_inner().challenge + ); + + let signed_challenge = keypair + .sign_message(formatted_challenge.as_bytes()) + .as_ref() + .to_vec(); + + debug!( + "formatted_challenge: {} signed_challenge: {:?}", + formatted_challenge, signed_challenge + ); + + debug!("generate_auth_tokens"); + let auth_tokens = auth_service_client + .generate_auth_tokens(GenerateAuthTokensRequest { + challenge: formatted_challenge, + client_pubkey: keypair.pubkey().as_ref().to_vec(), + signed_challenge, + }) + .await + .map_err(|e| ProxyError::AuthenticationError(e.to_string()))?; + + let inner = auth_tokens.into_inner(); + let access_token = get_validated_token(inner.access_token)?; + let refresh_token = get_validated_token(inner.refresh_token)?; + + Ok((access_token, refresh_token)) +} + +/// Tries to refresh the access token or run full-reauth if needed. +pub async fn maybe_refresh_auth_tokens( + auth_service_client: &mut AuthServiceClient, + access_token: &Arc>, + refresh_token: &Token, + cluster_info: &Arc, + connection_timeout: &Duration, + refresh_within_s: u64, +) -> crate::proxy::Result<( + Option, // access token + Option, // refresh token +)> { + let access_token_expiry: u64 = access_token + .lock() + .unwrap() + .expires_at_utc + .as_ref() + .map(|ts| ts.seconds as u64) + .unwrap_or_default(); + let refresh_token_expiry: u64 = refresh_token + .expires_at_utc + .as_ref() + .map(|ts| ts.seconds as u64) + .unwrap_or_default(); + + let now = Utc::now().timestamp() as u64; + + let should_refresh_access = + access_token_expiry.checked_sub(now).unwrap_or_default() <= refresh_within_s; + let should_generate_new_tokens = + refresh_token_expiry.checked_sub(now).unwrap_or_default() <= refresh_within_s; + + if should_generate_new_tokens { + let kp = cluster_info.keypair().clone(); + + let (new_access_token, new_refresh_token) = timeout( + *connection_timeout, + generate_auth_tokens(auth_service_client, kp.as_ref()), + ) + .await + .map_err(|_| ProxyError::MethodTimeout("generate_auth_tokens".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))?; + + return Ok((Some(new_access_token), Some(new_refresh_token))); + } else if should_refresh_access { + let new_access_token = timeout( + *connection_timeout, + refresh_access_token(auth_service_client, refresh_token), + ) + .await + .map_err(|_| ProxyError::MethodTimeout("refresh_access_token".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))?; + + return Ok((Some(new_access_token), None)); + } + + Ok((None, None)) +} + +pub async fn refresh_access_token( + auth_service_client: &mut AuthServiceClient, + refresh_token: &Token, +) -> crate::proxy::Result { + let response = auth_service_client + .refresh_access_token(RefreshAccessTokenRequest { + refresh_token: refresh_token.value.clone(), + }) + .await + .map_err(|e| ProxyError::AuthenticationError(e.to_string()))?; + get_validated_token(response.into_inner().access_token) +} + +/// An invalid token is one where any of its fields are None or the token itself is None. +/// Performs the necessary validations on the auth tokens before returning, +/// i.e. it is safe to call .unwrap() on the token fields from the call-site. +fn get_validated_token(maybe_token: Option) -> crate::proxy::Result { + let token = maybe_token + .ok_or_else(|| ProxyError::BadAuthenticationToken("received a null token".to_string()))?; + if token.expires_at_utc.is_none() { + Err(ProxyError::BadAuthenticationToken( + "expires_at_utc field is null".to_string(), + )) + } else { + Ok(token) + } +} diff --git a/core/src/proxy/block_engine_stage.rs b/core/src/proxy/block_engine_stage.rs new file mode 100644 index 0000000000..caf45ac0d9 --- /dev/null +++ b/core/src/proxy/block_engine_stage.rs @@ -0,0 +1,571 @@ +//! Maintains a connection to the Block Engine. +//! +//! The Block Engine is responsible for the following: +//! - Acts as a system that sends high profit bundles and transactions to a validator. +//! - Sends transactions and bundles to the validator. +use { + crate::{ + banking_trace::BankingPacketSender, + packet_bundle::PacketBundle, + proto_packet_to_packet, + proxy::{ + auth::{generate_auth_tokens, maybe_refresh_auth_tokens, AuthInterceptor}, + ProxyError, + }, + }, + crossbeam_channel::Sender, + jito_protos::proto::{ + auth::{auth_service_client::AuthServiceClient, Token}, + block_engine::{ + self, block_engine_validator_client::BlockEngineValidatorClient, + BlockBuilderFeeInfoRequest, + }, + }, + solana_gossip::cluster_info::ClusterInfo, + solana_perf::packet::PacketBatch, + solana_sdk::{ + pubkey::Pubkey, saturating_add_assign, signature::Signer, signer::keypair::Keypair, + }, + std::{ + str::FromStr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + thread::{self, Builder, JoinHandle}, + time::Duration, + }, + tokio::{ + task, + time::{interval, sleep, timeout}, + }, + tonic::{ + codegen::InterceptedService, + transport::{Channel, Endpoint}, + Status, Streaming, + }, +}; + +const CONNECTION_TIMEOUT_S: u64 = 10; +const CONNECTION_BACKOFF_S: u64 = 5; + +#[derive(Default)] +struct BlockEngineStageStats { + num_bundles: u64, + num_bundle_packets: u64, + num_packets: u64, + num_empty_packets: u64, +} + +impl BlockEngineStageStats { + pub(crate) fn report(&self) { + datapoint_info!( + "block_engine_stage-stats", + ("num_bundles", self.num_bundles, i64), + ("num_bundle_packets", self.num_bundle_packets, i64), + ("num_packets", self.num_packets, i64), + ("num_empty_packets", self.num_empty_packets, i64) + ); + } +} + +pub struct BlockBuilderFeeInfo { + pub block_builder: Pubkey, + pub block_builder_commission: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BlockEngineConfig { + /// Block Engine URL + pub block_engine_url: String, + + /// If set then it will be assumed the backend verified packets so signature verification will be bypassed in the validator. + pub trust_packets: bool, +} + +pub struct BlockEngineStage { + t_hdls: Vec>, +} + +impl BlockEngineStage { + pub fn new( + block_engine_config: Arc>, + // Channel that bundles get piped through. + bundle_tx: Sender>, + // The keypair stored here is used to sign auth challenges. + cluster_info: Arc, + // Channel that non-trusted packets get piped through. + packet_tx: Sender, + // Channel that trusted packets get piped through. + banking_packet_sender: BankingPacketSender, + exit: Arc, + block_builder_fee_info: &Arc>, + ) -> Self { + let block_builder_fee_info = block_builder_fee_info.clone(); + + let thread = Builder::new() + .name("block-engine-stage".to_string()) + .spawn(move || { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(Self::start( + block_engine_config, + cluster_info, + bundle_tx, + packet_tx, + banking_packet_sender, + exit, + block_builder_fee_info, + )); + }) + .unwrap(); + + Self { + t_hdls: vec![thread], + } + } + + pub fn join(self) -> thread::Result<()> { + for t in self.t_hdls { + t.join()?; + } + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + async fn start( + block_engine_config: Arc>, + cluster_info: Arc, + bundle_tx: Sender>, + packet_tx: Sender, + banking_packet_sender: BankingPacketSender, + exit: Arc, + block_builder_fee_info: Arc>, + ) { + const CONNECTION_TIMEOUT: Duration = Duration::from_secs(CONNECTION_TIMEOUT_S); + const CONNECTION_BACKOFF: Duration = Duration::from_secs(CONNECTION_BACKOFF_S); + let mut error_count: u64 = 0; + + while !exit.load(Ordering::Relaxed) { + // Wait until a valid config is supplied (either initially or by admin rpc) + // Use if!/else here to avoid extra CONNECTION_BACKOFF wait on successful termination + let local_block_engine_config = { + let block_engine_config = block_engine_config.clone(); + task::spawn_blocking(move || block_engine_config.lock().unwrap().clone()) + .await + .unwrap() + }; + if !Self::is_valid_block_engine_config(&local_block_engine_config) { + sleep(CONNECTION_BACKOFF).await; + } else if let Err(e) = Self::connect_auth_and_stream( + &local_block_engine_config, + &block_engine_config, + &cluster_info, + &bundle_tx, + &packet_tx, + &banking_packet_sender, + &exit, + &block_builder_fee_info, + &CONNECTION_TIMEOUT, + ) + .await + { + match e { + // This error is frequent on hot spares, and the parsed string does not work + // with datapoints (incorrect escaping). + ProxyError::AuthenticationPermissionDenied => { + warn!("block engine permission denied. not on leader schedule. ignore if hot-spare.") + } + e => { + error_count += 1; + datapoint_warn!( + "block_engine_stage-proxy_error", + ("count", error_count, i64), + ("error", e.to_string(), String), + ); + } + } + sleep(CONNECTION_BACKOFF).await; + } + } + } + + async fn connect_auth_and_stream( + local_block_engine_config: &BlockEngineConfig, + global_block_engine_config: &Arc>, + cluster_info: &Arc, + bundle_tx: &Sender>, + packet_tx: &Sender, + banking_packet_sender: &BankingPacketSender, + exit: &Arc, + block_builder_fee_info: &Arc>, + connection_timeout: &Duration, + ) -> crate::proxy::Result<()> { + // Get a copy of configs here in case they have changed at runtime + let keypair = cluster_info.keypair().clone(); + + let mut backend_endpoint = + Endpoint::from_shared(local_block_engine_config.block_engine_url.clone()) + .map_err(|_| { + ProxyError::BlockEngineConnectionError(format!( + "invalid block engine url value: {}", + local_block_engine_config.block_engine_url + )) + })? + .tcp_keepalive(Some(Duration::from_secs(60))); + if local_block_engine_config + .block_engine_url + .starts_with("https") + { + backend_endpoint = backend_endpoint + .tls_config(tonic::transport::ClientTlsConfig::new()) + .map_err(|_| { + ProxyError::BlockEngineConnectionError( + "failed to set tls_config for block engine service".to_string(), + ) + })?; + } + + debug!( + "connecting to auth: {}", + local_block_engine_config.block_engine_url + ); + let auth_channel = timeout(*connection_timeout, backend_endpoint.connect()) + .await + .map_err(|_| ProxyError::AuthenticationConnectionTimeout)? + .map_err(|e| ProxyError::AuthenticationConnectionError(e.to_string()))?; + + let mut auth_client = AuthServiceClient::new(auth_channel); + + debug!("generating authentication token"); + let (access_token, refresh_token) = timeout( + *connection_timeout, + generate_auth_tokens(&mut auth_client, &keypair), + ) + .await + .map_err(|_| ProxyError::AuthenticationTimeout)??; + + datapoint_info!( + "block_engine_stage-tokens_generated", + ("url", local_block_engine_config.block_engine_url, String), + ("count", 1, i64), + ); + + debug!( + "connecting to block engine: {}", + local_block_engine_config.block_engine_url + ); + let block_engine_channel = timeout(*connection_timeout, backend_endpoint.connect()) + .await + .map_err(|_| ProxyError::BlockEngineConnectionTimeout)? + .map_err(|e| ProxyError::BlockEngineConnectionError(e.to_string()))?; + + let access_token = Arc::new(Mutex::new(access_token)); + let block_engine_client = BlockEngineValidatorClient::with_interceptor( + block_engine_channel, + AuthInterceptor::new(access_token.clone()), + ); + + Self::start_consuming_block_engine_bundles_and_packets( + bundle_tx, + block_engine_client, + packet_tx, + local_block_engine_config, + global_block_engine_config, + banking_packet_sender, + exit, + block_builder_fee_info, + auth_client, + access_token, + refresh_token, + connection_timeout, + keypair, + cluster_info, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn start_consuming_block_engine_bundles_and_packets( + bundle_tx: &Sender>, + mut client: BlockEngineValidatorClient>, + packet_tx: &Sender, + local_config: &BlockEngineConfig, // local copy of config with current connections + global_config: &Arc>, // guarded reference for detecting run-time updates + banking_packet_sender: &BankingPacketSender, + exit: &Arc, + block_builder_fee_info: &Arc>, + auth_client: AuthServiceClient, + access_token: Arc>, + refresh_token: Token, + connection_timeout: &Duration, + keypair: Arc, + cluster_info: &Arc, + ) -> crate::proxy::Result<()> { + let subscribe_packets_stream = timeout( + *connection_timeout, + client.subscribe_packets(block_engine::SubscribePacketsRequest {}), + ) + .await + .map_err(|_| ProxyError::MethodTimeout("block_engine_subscribe_packets".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))? + .into_inner(); + + let subscribe_bundles_stream = timeout( + *connection_timeout, + client.subscribe_bundles(block_engine::SubscribeBundlesRequest {}), + ) + .await + .map_err(|_| ProxyError::MethodTimeout("subscribe_bundles".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))? + .into_inner(); + + let block_builder_info = timeout( + *connection_timeout, + client.get_block_builder_fee_info(BlockBuilderFeeInfoRequest {}), + ) + .await + .map_err(|_| ProxyError::MethodTimeout("get_block_builder_fee_info".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))? + .into_inner(); + + { + let block_builder_fee_info = block_builder_fee_info.clone(); + task::spawn_blocking(move || { + let mut bb_fee = block_builder_fee_info.lock().unwrap(); + bb_fee.block_builder_commission = block_builder_info.commission; + if let Ok(pk) = Pubkey::from_str(&block_builder_info.pubkey) { + bb_fee.block_builder = pk + } + }) + .await + .unwrap(); + } + + Self::consume_bundle_and_packet_stream( + client, + (subscribe_bundles_stream, subscribe_packets_stream), + bundle_tx, + packet_tx, + local_config, + global_config, + banking_packet_sender, + exit, + block_builder_fee_info, + auth_client, + access_token, + refresh_token, + keypair, + cluster_info, + connection_timeout, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn consume_bundle_and_packet_stream( + mut client: BlockEngineValidatorClient>, + (mut bundle_stream, mut packet_stream): ( + Streaming, + Streaming, + ), + bundle_tx: &Sender>, + packet_tx: &Sender, + local_config: &BlockEngineConfig, // local copy of config with current connections + global_config: &Arc>, // guarded reference for detecting run-time updates + banking_packet_sender: &BankingPacketSender, + exit: &Arc, + block_builder_fee_info: &Arc>, + mut auth_client: AuthServiceClient, + access_token: Arc>, + mut refresh_token: Token, + keypair: Arc, + cluster_info: &Arc, + connection_timeout: &Duration, + ) -> crate::proxy::Result<()> { + const METRICS_TICK: Duration = Duration::from_secs(1); + const MAINTENANCE_TICK: Duration = Duration::from_secs(10 * 60); + let refresh_within_s: u64 = METRICS_TICK.as_secs().saturating_mul(3).saturating_div(2); + + let mut num_full_refreshes: u64 = 1; + let mut num_refresh_access_token: u64 = 0; + let mut block_engine_stats = BlockEngineStageStats::default(); + let mut metrics_and_auth_tick = interval(METRICS_TICK); + let mut maintenance_tick = interval(MAINTENANCE_TICK); + + info!("connected to packet and bundle stream"); + + while !exit.load(Ordering::Relaxed) { + tokio::select! { + maybe_msg = packet_stream.message() => { + let resp = maybe_msg?.ok_or(ProxyError::GrpcStreamDisconnected)?; + Self::handle_block_engine_packets(resp, packet_tx, banking_packet_sender, local_config.trust_packets, &mut block_engine_stats)?; + } + maybe_bundles = bundle_stream.message() => { + Self::handle_block_engine_maybe_bundles(maybe_bundles, bundle_tx, &mut block_engine_stats)?; + } + _ = metrics_and_auth_tick.tick() => { + block_engine_stats.report(); + block_engine_stats = BlockEngineStageStats::default(); + + if cluster_info.id() != keypair.pubkey() { + return Err(ProxyError::AuthenticationConnectionError("validator identity changed".to_string())); + } + + let global_config = global_config.clone(); + if *local_config != task::spawn_blocking(move || global_config.lock().unwrap().clone()) + .await + .unwrap() { + return Err(ProxyError::AuthenticationConnectionError("block engine config changed".to_string())); + } + + let (maybe_new_access, maybe_new_refresh) = maybe_refresh_auth_tokens(&mut auth_client, + &access_token, + &refresh_token, + cluster_info, + connection_timeout, + refresh_within_s, + ).await?; + + if let Some(new_token) = maybe_new_access { + num_refresh_access_token += 1; + datapoint_info!( + "block_engine_stage-refresh_access_token", + ("url", &local_config.block_engine_url, String), + ("count", num_refresh_access_token, i64), + ); + + let access_token = access_token.clone(); + task::spawn_blocking(move || *access_token.lock().unwrap() = new_token) + .await + .unwrap(); + } + if let Some(new_token) = maybe_new_refresh { + num_full_refreshes += 1; + datapoint_info!( + "block_engine_stage-tokens_generated", + ("url", &local_config.block_engine_url, String), + ("count", num_full_refreshes, i64), + ); + refresh_token = new_token; + } + } + _ = maintenance_tick.tick() => { + let block_builder_info = timeout( + *connection_timeout, + client.get_block_builder_fee_info(BlockBuilderFeeInfoRequest{}) + ) + .await + .map_err(|_| ProxyError::MethodTimeout("get_block_builder_fee_info".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))? + .into_inner(); + + let block_builder_fee_info = block_builder_fee_info.clone(); + task::spawn_blocking(move || { + let mut bb_fee = block_builder_fee_info.lock().unwrap(); + bb_fee.block_builder_commission = block_builder_info.commission; + if let Ok(pk) = Pubkey::from_str(&block_builder_info.pubkey) { + bb_fee.block_builder = pk + } + }) + .await + .unwrap(); + } + } + } + + Ok(()) + } + + fn handle_block_engine_maybe_bundles( + maybe_bundles_response: Result, Status>, + bundle_sender: &Sender>, + block_engine_stats: &mut BlockEngineStageStats, + ) -> crate::proxy::Result<()> { + let bundles_response = maybe_bundles_response?.ok_or(ProxyError::GrpcStreamDisconnected)?; + let bundles: Vec = bundles_response + .bundles + .into_iter() + .filter_map(|bundle| { + Some(PacketBundle { + batch: PacketBatch::new( + bundle + .bundle? + .packets + .into_iter() + .map(proto_packet_to_packet) + .collect(), + ), + bundle_id: bundle.uuid, + }) + }) + .collect(); + + saturating_add_assign!(block_engine_stats.num_bundles, bundles.len() as u64); + saturating_add_assign!( + block_engine_stats.num_bundle_packets, + bundles.iter().map(|bundle| bundle.batch.len() as u64).sum() + ); + + // NOTE: bundles are sanitized in bundle_sanitizer module + bundle_sender + .send(bundles) + .map_err(|_| ProxyError::PacketForwardError) + } + + fn handle_block_engine_packets( + resp: block_engine::SubscribePacketsResponse, + packet_tx: &Sender, + banking_packet_sender: &BankingPacketSender, + trust_packets: bool, + block_engine_stats: &mut BlockEngineStageStats, + ) -> crate::proxy::Result<()> { + if let Some(batch) = resp.batch { + if batch.packets.is_empty() { + saturating_add_assign!(block_engine_stats.num_empty_packets, 1); + return Ok(()); + } + + let packet_batch = PacketBatch::new( + batch + .packets + .into_iter() + .map(proto_packet_to_packet) + .collect(), + ); + + saturating_add_assign!(block_engine_stats.num_packets, packet_batch.len() as u64); + + if trust_packets { + banking_packet_sender + .send(Arc::new((vec![packet_batch], None))) + .map_err(|_| ProxyError::PacketForwardError)?; + } else { + packet_tx + .send(packet_batch) + .map_err(|_| ProxyError::PacketForwardError)?; + } + } else { + saturating_add_assign!(block_engine_stats.num_empty_packets, 1); + } + + Ok(()) + } + + pub fn is_valid_block_engine_config(config: &BlockEngineConfig) -> bool { + if config.block_engine_url.is_empty() { + warn!("can't connect to block_engine. missing block_engine_url."); + return false; + } + if let Err(e) = Endpoint::from_str(&config.block_engine_url) { + error!( + "can't connect to block engine. error creating block engine endpoint - {}", + e.to_string() + ); + return false; + } + true + } +} diff --git a/core/src/proxy/fetch_stage_manager.rs b/core/src/proxy/fetch_stage_manager.rs new file mode 100644 index 0000000000..0d26c001a7 --- /dev/null +++ b/core/src/proxy/fetch_stage_manager.rs @@ -0,0 +1,170 @@ +use { + crate::proxy::{HeartbeatEvent, ProxyError}, + crossbeam_channel::{select, tick, Receiver, Sender}, + solana_client::connection_cache::Protocol, + solana_gossip::{cluster_info::ClusterInfo, contact_info}, + solana_perf::packet::PacketBatch, + std::{ + net::SocketAddr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread::{self, Builder, JoinHandle}, + time::{Duration, Instant}, + }, +}; + +const HEARTBEAT_TIMEOUT: Duration = Duration::from_millis(1500); // Empirically determined from load testing +const DISCONNECT_DELAY: Duration = Duration::from_secs(60); +const METRICS_CADENCE: Duration = Duration::from_secs(1); + +/// Manages switching between the validator's tpu ports and that of the proxy's. +/// Switch-overs are triggered by late and missed heartbeats. +pub struct FetchStageManager { + t_hdl: JoinHandle<()>, +} + +impl FetchStageManager { + pub fn new( + // ClusterInfo is used to switch between advertising the proxy's TPU ports and that of this validator's. + cluster_info: Arc, + // Channel that heartbeats are received from. Entirely responsible for triggering switch-overs. + heartbeat_rx: Receiver, + // Channel that packets from FetchStage are intercepted from. + packet_intercept_rx: Receiver, + // Intercepted packets get piped through here. + packet_tx: Sender, + exit: Arc, + ) -> Self { + let t_hdl = Self::start( + cluster_info, + heartbeat_rx, + packet_intercept_rx, + packet_tx, + exit, + ); + + Self { t_hdl } + } + + /// Disconnect fetch behaviour + /// Starts connected + /// When connected and a packet is received, forward it + /// When disconnected, packet is dropped + /// When receiving heartbeat while connected and not pending disconnect + /// Sets pending_disconnect to true and records time + /// When receiving heartbeat while connected, and pending for > DISCONNECT_DELAY_SEC + /// Sets fetch_connected to false, pending_disconnect to false + /// Advertises TPU ports sent in heartbeat + /// When tick is received without heartbeat_received + /// Sets fetch_connected to true, pending_disconnect to false + /// Advertises saved contact info + fn start( + cluster_info: Arc, + heartbeat_rx: Receiver, + packet_intercept_rx: Receiver, + packet_tx: Sender, + exit: Arc, + ) -> JoinHandle<()> { + Builder::new().name("fetch-stage-manager".into()).spawn(move || { + let my_fallback_contact_info = cluster_info.my_contact_info(); + + let mut fetch_connected = true; + let mut heartbeat_received = false; + let mut pending_disconnect = false; + + let mut pending_disconnect_ts = Instant::now(); + + let heartbeat_tick = tick(HEARTBEAT_TIMEOUT); + let metrics_tick = tick(METRICS_CADENCE); + let mut packets_forwarded = 0; + let mut heartbeats_received = 0; + loop { + select! { + recv(packet_intercept_rx) -> pkt => { + match pkt { + Ok(pkt) => { + if fetch_connected { + if packet_tx.send(pkt).is_err() { + error!("{:?}", ProxyError::PacketForwardError); + return; + } + packets_forwarded += 1; + } + } + Err(_) => { + warn!("packet intercept receiver disconnected, shutting down"); + return; + } + } + } + recv(heartbeat_tick) -> _ => { + if exit.load(Ordering::Relaxed) { + break; + } + if !heartbeat_received && (!fetch_connected || pending_disconnect) { + warn!("heartbeat late, reconnecting fetch stage"); + fetch_connected = true; + pending_disconnect = false; + + // yes, using UDP here is extremely confusing for the validator + // since the entire network is running QUIC. However, it's correct. + if let Err(e) = Self::set_tpu_addresses(&cluster_info, my_fallback_contact_info.tpu(Protocol::UDP).unwrap(), my_fallback_contact_info.tpu_forwards(Protocol::UDP).unwrap()) { + error!("error setting tpu or tpu_fwd to ({:?}, {:?}), error: {:?}", my_fallback_contact_info.tpu(Protocol::UDP).unwrap(), my_fallback_contact_info.tpu_forwards(Protocol::UDP).unwrap(), e); + } + heartbeats_received = 0; + } + heartbeat_received = false; + } + recv(heartbeat_rx) -> tpu_info => { + if let Ok((tpu_addr, tpu_forward_addr)) = tpu_info { + heartbeats_received += 1; + heartbeat_received = true; + if fetch_connected && !pending_disconnect { + info!("received heartbeat while fetch stage connected, pending disconnect after delay"); + pending_disconnect_ts = Instant::now(); + pending_disconnect = true; + } + if fetch_connected && pending_disconnect && pending_disconnect_ts.elapsed() > DISCONNECT_DELAY { + info!("disconnecting fetch stage"); + fetch_connected = false; + pending_disconnect = false; + if let Err(e) = Self::set_tpu_addresses(&cluster_info, tpu_addr, tpu_forward_addr) { + error!("error setting tpu or tpu_fwd to ({:?}, {:?}), error: {:?}", tpu_addr, tpu_forward_addr, e); + } + } + } else { + { + warn!("relayer heartbeat receiver disconnected, shutting down"); + return; + } + } + } + recv(metrics_tick) -> _ => { + datapoint_info!( + "relayer-heartbeat", + ("fetch_stage_packets_forwarded", packets_forwarded, i64), + ("heartbeats_received", heartbeats_received, i64), + ); + + } + } + } + }).unwrap() + } + + fn set_tpu_addresses( + cluster_info: &Arc, + tpu_address: SocketAddr, + tpu_forward_address: SocketAddr, + ) -> Result<(), contact_info::Error> { + cluster_info.set_tpu(tpu_address)?; + cluster_info.set_tpu_forwards(tpu_forward_address)?; + Ok(()) + } + + pub fn join(self) -> thread::Result<()> { + self.t_hdl.join() + } +} diff --git a/core/src/proxy/mod.rs b/core/src/proxy/mod.rs new file mode 100644 index 0000000000..86d48482aa --- /dev/null +++ b/core/src/proxy/mod.rs @@ -0,0 +1,100 @@ +//! This module contains logic for connecting to an external Relayer and Block Engine. +//! The Relayer acts as an external TPU and TPU Forward socket while the Block Engine +//! is tasked with streaming high value bundles to the validator. The validator can run +//! in one of 3 modes: +//! 1. Connected to Relayer and Block Engine. +//! - This is the ideal mode as it increases the probability of building the most profitable blocks. +//! 2. Connected only to Relayer. +//! - A validator may choose to run in this mode if the main concern is to offload ingress traffic deduplication and sig-verification. +//! 3. Connected only to Block Engine. +//! - Running in this mode means pending transactions are not exposed to external actors. This mode is ideal if the validator wishes +//! to accept bundles while maintaining some level of privacy for in-flight transactions. + +mod auth; +pub mod block_engine_stage; +pub mod fetch_stage_manager; +pub mod relayer_stage; + +use { + std::{ + net::{AddrParseError, SocketAddr}, + result, + }, + thiserror::Error, + tonic::Status, +}; + +type Result = result::Result; +type HeartbeatEvent = (SocketAddr, SocketAddr); + +#[derive(Error, Debug)] +pub enum ProxyError { + #[error("grpc error: {0}")] + GrpcError(#[from] Status), + + #[error("stream disconnected")] + GrpcStreamDisconnected, + + #[error("heartbeat error")] + HeartbeatChannelError, + + #[error("heartbeat expired")] + HeartbeatExpired, + + #[error("error forwarding packet to banking stage")] + PacketForwardError, + + #[error("missing tpu config: {0:?}")] + MissingTpuSocket(String), + + #[error("invalid socket address: {0:?}")] + InvalidSocketAddress(#[from] AddrParseError), + + #[error("invalid gRPC data: {0:?}")] + InvalidData(String), + + #[error("timeout: {0:?}")] + ConnectionError(#[from] tonic::transport::Error), + + #[error("AuthenticationConnectionTimeout")] + AuthenticationConnectionTimeout, + + #[error("AuthenticationTimeout")] + AuthenticationTimeout, + + #[error("AuthenticationConnectionError: {0:?}")] + AuthenticationConnectionError(String), + + #[error("BlockEngineConnectionTimeout")] + BlockEngineConnectionTimeout, + + #[error("BlockEngineTimeout")] + BlockEngineTimeout, + + #[error("BlockEngineConnectionError: {0:?}")] + BlockEngineConnectionError(String), + + #[error("RelayerConnectionTimeout")] + RelayerConnectionTimeout, + + #[error("RelayerTimeout")] + RelayerEngineTimeout, + + #[error("RelayerConnectionError: {0:?}")] + RelayerConnectionError(String), + + #[error("AuthenticationError: {0:?}")] + AuthenticationError(String), + + #[error("AuthenticationPermissionDenied")] + AuthenticationPermissionDenied, + + #[error("BadAuthenticationToken: {0:?}")] + BadAuthenticationToken(String), + + #[error("MethodTimeout: {0:?}")] + MethodTimeout(String), + + #[error("MethodError: {0:?}")] + MethodError(String), +} diff --git a/core/src/proxy/relayer_stage.rs b/core/src/proxy/relayer_stage.rs new file mode 100644 index 0000000000..0c8ce22877 --- /dev/null +++ b/core/src/proxy/relayer_stage.rs @@ -0,0 +1,515 @@ +//! Maintains a connection to the Relayer. +//! +//! The external Relayer is responsible for the following: +//! - Acts as a TPU proxy. +//! - Sends transactions to the validator. +//! - Does not bundles to avoid DOS vector. +//! - When validator connects, it changes its TPU and TPU forward address to the relayer. +//! - Expected to send heartbeat to validator as watchdog. If watchdog times out, the validator +//! disconnects and reverts the TPU and TPU forward settings. + +use { + crate::{ + banking_trace::BankingPacketSender, + proto_packet_to_packet, + proxy::{ + auth::{generate_auth_tokens, maybe_refresh_auth_tokens, AuthInterceptor}, + HeartbeatEvent, ProxyError, + }, + }, + crossbeam_channel::Sender, + jito_protos::proto::{ + auth::{auth_service_client::AuthServiceClient, Token}, + relayer::{self, relayer_client::RelayerClient}, + }, + solana_gossip::cluster_info::ClusterInfo, + solana_perf::packet::PacketBatch, + solana_sdk::{ + saturating_add_assign, + signature::{Keypair, Signer}, + }, + std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + str::FromStr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + thread::{self, Builder, JoinHandle}, + time::{Duration, Instant}, + }, + tokio::{ + task, + time::{interval, sleep, timeout}, + }, + tonic::{ + codegen::InterceptedService, + transport::{Channel, Endpoint}, + Streaming, + }, +}; + +const CONNECTION_TIMEOUT_S: u64 = 10; +const CONNECTION_BACKOFF_S: u64 = 5; + +#[derive(Default)] +struct RelayerStageStats { + num_empty_messages: u64, + num_packets: u64, + num_heartbeats: u64, +} + +impl RelayerStageStats { + pub(crate) fn report(&self) { + datapoint_info!( + "relayer_stage-stats", + ("num_empty_messages", self.num_empty_messages, i64), + ("num_packets", self.num_packets, i64), + ("num_heartbeats", self.num_heartbeats, i64), + ); + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RelayerConfig { + /// Relayer URL + pub relayer_url: String, + + /// Interval at which heartbeats are expected. + pub expected_heartbeat_interval: Duration, + + /// The max tolerable age of the last heartbeat. + pub oldest_allowed_heartbeat: Duration, + + /// If set then it will be assumed the backend verified packets so signature verification will be bypassed in the validator. + pub trust_packets: bool, +} + +pub struct RelayerStage { + t_hdls: Vec>, +} + +impl RelayerStage { + pub fn new( + relayer_config: Arc>, + // The keypair stored here is used to sign auth challenges. + cluster_info: Arc, + // Channel that server-sent heartbeats are piped through. + heartbeat_tx: Sender, + // Channel that non-trusted streamed packets are piped through. + packet_tx: Sender, + // Channel that trusted streamed packets are piped through. + banking_packet_sender: BankingPacketSender, + exit: Arc, + ) -> Self { + let thread = Builder::new() + .name("relayer-stage".to_string()) + .spawn(move || { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + rt.block_on(Self::start( + relayer_config, + cluster_info, + heartbeat_tx, + packet_tx, + banking_packet_sender, + exit, + )); + }) + .unwrap(); + + Self { + t_hdls: vec![thread], + } + } + + pub fn join(self) -> thread::Result<()> { + for t in self.t_hdls { + t.join()?; + } + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + async fn start( + relayer_config: Arc>, + cluster_info: Arc, + heartbeat_tx: Sender, + packet_tx: Sender, + banking_packet_sender: BankingPacketSender, + exit: Arc, + ) { + const CONNECTION_TIMEOUT: Duration = Duration::from_secs(CONNECTION_TIMEOUT_S); + const CONNECTION_BACKOFF: Duration = Duration::from_secs(CONNECTION_BACKOFF_S); + + let mut error_count: u64 = 0; + + while !exit.load(Ordering::Relaxed) { + // Wait until a valid config is supplied (either initially or by admin rpc) + // Use if!/else here to avoid extra CONNECTION_BACKOFF wait on successful termination + let local_relayer_config = { + let relayer_config = relayer_config.clone(); + task::spawn_blocking(move || relayer_config.lock().unwrap().clone()) + .await + .expect("Failed to get execute tokio task.") + }; + if !Self::is_valid_relayer_config(&local_relayer_config) { + sleep(CONNECTION_BACKOFF).await; + } else if let Err(e) = Self::connect_auth_and_stream( + &local_relayer_config, + &relayer_config, + &cluster_info, + &heartbeat_tx, + &packet_tx, + &banking_packet_sender, + &exit, + &CONNECTION_TIMEOUT, + ) + .await + { + match e { + // This error is frequent on hot spares, and the parsed string does not work + // with datapoints (incorrect escaping). + ProxyError::AuthenticationPermissionDenied => { + warn!("relayer permission denied. not on leader schedule. ignore if hot-spare.") + } + e => { + error_count += 1; + datapoint_warn!( + "relayer_stage-proxy_error", + ("count", error_count, i64), + ("error", e.to_string(), String), + ); + } + } + sleep(CONNECTION_BACKOFF).await; + } + } + } + + async fn connect_auth_and_stream( + local_relayer_config: &RelayerConfig, + global_relayer_config: &Arc>, + cluster_info: &Arc, + heartbeat_tx: &Sender, + packet_tx: &Sender, + banking_packet_sender: &BankingPacketSender, + exit: &Arc, + connection_timeout: &Duration, + ) -> crate::proxy::Result<()> { + // Get a copy of configs here in case they have changed at runtime + let keypair = cluster_info.keypair().clone(); + + let mut backend_endpoint = Endpoint::from_shared(local_relayer_config.relayer_url.clone()) + .map_err(|_| { + ProxyError::RelayerConnectionError(format!( + "invalid relayer url value: {}", + local_relayer_config.relayer_url + )) + })? + .tcp_keepalive(Some(Duration::from_secs(60))); + if local_relayer_config.relayer_url.starts_with("https") { + backend_endpoint = backend_endpoint + .tls_config(tonic::transport::ClientTlsConfig::new()) + .map_err(|_| { + ProxyError::RelayerConnectionError( + "failed to set tls_config for relayer service".to_string(), + ) + })?; + } + + debug!("connecting to auth: {}", local_relayer_config.relayer_url); + let auth_channel = timeout(*connection_timeout, backend_endpoint.connect()) + .await + .map_err(|_| ProxyError::AuthenticationConnectionTimeout)? + .map_err(|e| ProxyError::AuthenticationConnectionError(e.to_string()))?; + + let mut auth_client = AuthServiceClient::new(auth_channel); + + debug!("generating authentication token"); + let (access_token, refresh_token) = timeout( + *connection_timeout, + generate_auth_tokens(&mut auth_client, &keypair), + ) + .await + .map_err(|_| ProxyError::AuthenticationTimeout)??; + + datapoint_info!( + "relayer_stage-tokens_generated", + ("url", local_relayer_config.relayer_url, String), + ("count", 1, i64), + ); + + debug!( + "connecting to relayer: {}", + local_relayer_config.relayer_url + ); + let relayer_channel = timeout(*connection_timeout, backend_endpoint.connect()) + .await + .map_err(|_| ProxyError::RelayerConnectionTimeout)? + .map_err(|e| ProxyError::RelayerConnectionError(e.to_string()))?; + + let access_token = Arc::new(Mutex::new(access_token)); + let relayer_client = RelayerClient::with_interceptor( + relayer_channel, + AuthInterceptor::new(access_token.clone()), + ); + + Self::start_consuming_relayer_packets( + relayer_client, + heartbeat_tx, + packet_tx, + banking_packet_sender, + local_relayer_config, + global_relayer_config, + exit, + auth_client, + access_token, + refresh_token, + keypair, + cluster_info, + connection_timeout, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn start_consuming_relayer_packets( + mut client: RelayerClient>, + heartbeat_tx: &Sender, + packet_tx: &Sender, + banking_packet_sender: &BankingPacketSender, + local_config: &RelayerConfig, // local copy of config with current connections + global_config: &Arc>, // guarded reference for detecting run-time updates + exit: &Arc, + auth_client: AuthServiceClient, + access_token: Arc>, + refresh_token: Token, + keypair: Arc, + cluster_info: &Arc, + connection_timeout: &Duration, + ) -> crate::proxy::Result<()> { + let heartbeat_event: HeartbeatEvent = { + let tpu_config = timeout( + *connection_timeout, + client.get_tpu_configs(relayer::GetTpuConfigsRequest {}), + ) + .await + .map_err(|_| ProxyError::MethodTimeout("relayer_get_tpu_configs".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))? + .into_inner(); + + let tpu_addr = tpu_config + .tpu + .ok_or_else(|| ProxyError::MissingTpuSocket("tpu".to_string()))?; + let tpu_forward_addr = tpu_config + .tpu_forward + .ok_or_else(|| ProxyError::MissingTpuSocket("tpu_fwd".to_string()))?; + + let tpu_ip = IpAddr::from(tpu_addr.ip.parse::()?); + let tpu_forward_ip = IpAddr::from(tpu_forward_addr.ip.parse::()?); + + let tpu_socket = SocketAddr::new(tpu_ip, tpu_addr.port as u16); + let tpu_forward_socket = SocketAddr::new(tpu_forward_ip, tpu_forward_addr.port as u16); + (tpu_socket, tpu_forward_socket) + }; + + let packet_stream = timeout( + *connection_timeout, + client.subscribe_packets(relayer::SubscribePacketsRequest {}), + ) + .await + .map_err(|_| ProxyError::MethodTimeout("relayer_subscribe_packets".to_string()))? + .map_err(|e| ProxyError::MethodError(e.to_string()))? + .into_inner(); + + Self::consume_packet_stream( + heartbeat_event, + heartbeat_tx, + packet_stream, + packet_tx, + local_config, + global_config, + banking_packet_sender, + exit, + auth_client, + access_token, + refresh_token, + keypair, + cluster_info, + connection_timeout, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn consume_packet_stream( + heartbeat_event: HeartbeatEvent, + heartbeat_tx: &Sender, + mut packet_stream: Streaming, + packet_tx: &Sender, + local_config: &RelayerConfig, // local copy of config with current connections + global_config: &Arc>, // guarded reference for detecting run-time updates + banking_packet_sender: &BankingPacketSender, + exit: &Arc, + mut auth_client: AuthServiceClient, + access_token: Arc>, + mut refresh_token: Token, + keypair: Arc, + cluster_info: &Arc, + connection_timeout: &Duration, + ) -> crate::proxy::Result<()> { + const METRICS_TICK: Duration = Duration::from_secs(1); + let refresh_within_s: u64 = METRICS_TICK.as_secs().saturating_mul(3).saturating_div(2); + + let mut relayer_stats = RelayerStageStats::default(); + let mut metrics_and_auth_tick = interval(METRICS_TICK); + + let mut num_full_refreshes: u64 = 1; + let mut num_refresh_access_token: u64 = 0; + + let mut heartbeat_check_interval = interval(local_config.expected_heartbeat_interval); + let mut last_heartbeat_ts = Instant::now(); + + info!("connected to packet stream"); + + while !exit.load(Ordering::Relaxed) { + tokio::select! { + maybe_msg = packet_stream.message() => { + let resp = maybe_msg?.ok_or(ProxyError::GrpcStreamDisconnected)?; + Self::handle_relayer_packets(resp, heartbeat_event, heartbeat_tx, &mut last_heartbeat_ts, packet_tx, local_config.trust_packets, banking_packet_sender, &mut relayer_stats)?; + } + _ = heartbeat_check_interval.tick() => { + if last_heartbeat_ts.elapsed() > local_config.oldest_allowed_heartbeat { + return Err(ProxyError::HeartbeatExpired); + } + } + _ = metrics_and_auth_tick.tick() => { + relayer_stats.report(); + relayer_stats = RelayerStageStats::default(); + + if cluster_info.id() != keypair.pubkey() { + return Err(ProxyError::AuthenticationConnectionError("validator identity changed".to_string())); + } + + let global_config = global_config.clone(); + if *local_config != task::spawn_blocking(move || global_config.lock().unwrap().clone()) + .await + .unwrap() { + return Err(ProxyError::AuthenticationConnectionError("relayer config changed".to_string())); + } + + let (maybe_new_access, maybe_new_refresh) = maybe_refresh_auth_tokens(&mut auth_client, + &access_token, + &refresh_token, + cluster_info, + connection_timeout, + refresh_within_s, + ).await?; + + if let Some(new_token) = maybe_new_access { + num_refresh_access_token += 1; + datapoint_info!( + "relayer_stage-refresh_access_token", + ("url", &local_config.relayer_url, String), + ("count", num_refresh_access_token, i64), + ); + + let access_token = access_token.clone(); + task::spawn_blocking(move || *access_token.lock().unwrap() = new_token) + .await + .unwrap(); + } + if let Some(new_token) = maybe_new_refresh { + num_full_refreshes += 1; + datapoint_info!( + "relayer_stage-tokens_generated", + ("url", &local_config.relayer_url, String), + ("count", num_full_refreshes, i64), + ); + refresh_token = new_token; + } + } + } + } + Ok(()) + } + + fn handle_relayer_packets( + subscribe_packets_resp: relayer::SubscribePacketsResponse, + heartbeat_event: HeartbeatEvent, + heartbeat_tx: &Sender, + last_heartbeat_ts: &mut Instant, + packet_tx: &Sender, + trust_packets: bool, + banking_packet_sender: &BankingPacketSender, + relayer_stats: &mut RelayerStageStats, + ) -> crate::proxy::Result<()> { + match subscribe_packets_resp.msg { + None => { + saturating_add_assign!(relayer_stats.num_empty_messages, 1); + } + Some(relayer::subscribe_packets_response::Msg::Batch(proto_batch)) => { + if proto_batch.packets.is_empty() { + saturating_add_assign!(relayer_stats.num_empty_messages, 1); + return Ok(()); + } + + let packet_batch = PacketBatch::new( + proto_batch + .packets + .into_iter() + .map(proto_packet_to_packet) + .collect(), + ); + + saturating_add_assign!(relayer_stats.num_packets, packet_batch.len() as u64); + + if trust_packets { + banking_packet_sender + .send(Arc::new((vec![packet_batch], None))) + .map_err(|_| ProxyError::PacketForwardError)?; + } else { + packet_tx + .send(packet_batch) + .map_err(|_| ProxyError::PacketForwardError)?; + } + } + Some(relayer::subscribe_packets_response::Msg::Heartbeat(_)) => { + saturating_add_assign!(relayer_stats.num_heartbeats, 1); + + *last_heartbeat_ts = Instant::now(); + heartbeat_tx + .send(heartbeat_event) + .map_err(|_| ProxyError::HeartbeatChannelError)?; + } + } + Ok(()) + } + + pub fn is_valid_relayer_config(config: &RelayerConfig) -> bool { + if config.relayer_url.is_empty() { + warn!("can't connect to relayer. missing relayer_url."); + return false; + } + if config.oldest_allowed_heartbeat.is_zero() { + error!("can't connect to relayer. oldest allowed heartbeat must be greater than 0."); + return false; + } + if config.expected_heartbeat_interval.is_zero() { + error!("can't connect to relayer. expected heartbeat interval must be greater than 0."); + return false; + } + if let Err(e) = Endpoint::from_str(&config.relayer_url) { + error!( + "can't connect to relayer. error creating relayer endpoint - {}", + e.to_string() + ); + return false; + } + true + } +} diff --git a/core/src/tip_manager.rs b/core/src/tip_manager.rs new file mode 100644 index 0000000000..397dfffea1 --- /dev/null +++ b/core/src/tip_manager.rs @@ -0,0 +1,588 @@ +use { + crate::proxy::block_engine_stage::BlockBuilderFeeInfo, + anchor_lang::{AccountDeserialize, InstructionData, ToAccountMetas}, + jito_tip_distribution::sdk::{ + derive_config_account_address, derive_tip_distribution_account_address, + instruction::{ + initialize_ix, initialize_tip_distribution_account_ix, InitializeAccounts, + InitializeArgs, InitializeTipDistributionAccountAccounts, + InitializeTipDistributionAccountArgs, + }, + }, + jito_tip_payment::{ + Config, InitBumps, TipPaymentAccount, CONFIG_ACCOUNT_SEED, TIP_ACCOUNT_SEED_0, + TIP_ACCOUNT_SEED_1, TIP_ACCOUNT_SEED_2, TIP_ACCOUNT_SEED_3, TIP_ACCOUNT_SEED_4, + TIP_ACCOUNT_SEED_5, TIP_ACCOUNT_SEED_6, TIP_ACCOUNT_SEED_7, + }, + log::warn, + solana_bundle::TipError, + solana_runtime::bank::Bank, + solana_sdk::{ + account::ReadableAccount, + bundle::{derive_bundle_id_from_sanitized_transactions, SanitizedBundle}, + instruction::Instruction, + pubkey::Pubkey, + signature::Keypair, + signer::Signer, + stake_history::Epoch, + system_program, + transaction::{SanitizedTransaction, Transaction}, + }, + std::{collections::HashSet, sync::Arc}, +}; + +pub type Result = std::result::Result; + +#[derive(Debug, Clone)] +struct TipPaymentProgramInfo { + program_id: Pubkey, + + config_pda_bump: (Pubkey, u8), + tip_pda_0: (Pubkey, u8), + tip_pda_1: (Pubkey, u8), + tip_pda_2: (Pubkey, u8), + tip_pda_3: (Pubkey, u8), + tip_pda_4: (Pubkey, u8), + tip_pda_5: (Pubkey, u8), + tip_pda_6: (Pubkey, u8), + tip_pda_7: (Pubkey, u8), +} + +/// Contains metadata regarding the tip-distribution account. +/// The PDAs contained in this struct are presumed to be owned by the program. +#[derive(Debug, Clone)] +struct TipDistributionProgramInfo { + /// The tip-distribution program_id. + program_id: Pubkey, + + /// Singleton [Config] PDA and bump tuple. + config_pda_and_bump: (Pubkey, u8), +} + +/// This config is used on each invocation to the `initialize_tip_distribution_account` instruction. +#[derive(Debug, Clone)] +pub struct TipDistributionAccountConfig { + /// The account with authority to upload merkle-roots to this validator's [TipDistributionAccount]. + pub merkle_root_upload_authority: Pubkey, + + /// This validator's vote account. + pub vote_account: Pubkey, + + /// This validator's commission rate BPS for tips in the [TipDistributionAccount]. + pub commission_bps: u16, +} + +impl Default for TipDistributionAccountConfig { + fn default() -> Self { + Self { + merkle_root_upload_authority: Pubkey::new_unique(), + vote_account: Pubkey::new_unique(), + commission_bps: 0, + } + } +} + +#[derive(Debug, Clone)] +pub struct TipManager { + tip_payment_program_info: TipPaymentProgramInfo, + tip_distribution_program_info: TipDistributionProgramInfo, + tip_distribution_account_config: TipDistributionAccountConfig, +} + +#[derive(Clone)] +pub struct TipManagerConfig { + pub tip_payment_program_id: Pubkey, + pub tip_distribution_program_id: Pubkey, + pub tip_distribution_account_config: TipDistributionAccountConfig, +} + +impl Default for TipManagerConfig { + fn default() -> Self { + TipManagerConfig { + tip_payment_program_id: Pubkey::new_unique(), + tip_distribution_program_id: Pubkey::new_unique(), + tip_distribution_account_config: TipDistributionAccountConfig::default(), + } + } +} + +impl TipManager { + pub fn new(config: TipManagerConfig) -> TipManager { + let TipManagerConfig { + tip_payment_program_id, + tip_distribution_program_id, + tip_distribution_account_config, + } = config; + + let config_pda_bump = + Pubkey::find_program_address(&[CONFIG_ACCOUNT_SEED], &tip_payment_program_id); + + let tip_pda_0 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_0], &tip_payment_program_id); + let tip_pda_1 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_1], &tip_payment_program_id); + let tip_pda_2 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_2], &tip_payment_program_id); + let tip_pda_3 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_3], &tip_payment_program_id); + let tip_pda_4 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_4], &tip_payment_program_id); + let tip_pda_5 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_5], &tip_payment_program_id); + let tip_pda_6 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_6], &tip_payment_program_id); + let tip_pda_7 = + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_7], &tip_payment_program_id); + + let config_pda_and_bump = derive_config_account_address(&tip_distribution_program_id); + + TipManager { + tip_payment_program_info: TipPaymentProgramInfo { + program_id: tip_payment_program_id, + config_pda_bump, + tip_pda_0, + tip_pda_1, + tip_pda_2, + tip_pda_3, + tip_pda_4, + tip_pda_5, + tip_pda_6, + tip_pda_7, + }, + tip_distribution_program_info: TipDistributionProgramInfo { + program_id: tip_distribution_program_id, + config_pda_and_bump, + }, + tip_distribution_account_config, + } + } + + pub fn tip_payment_program_id(&self) -> Pubkey { + self.tip_payment_program_info.program_id + } + + pub fn tip_distribution_program_id(&self) -> Pubkey { + self.tip_distribution_program_info.program_id + } + + /// Returns the [Config] account owned by the tip-payment program. + pub fn tip_payment_config_pubkey(&self) -> Pubkey { + self.tip_payment_program_info.config_pda_bump.0 + } + + /// Returns the [Config] account owned by the tip-distribution program. + pub fn tip_distribution_config_pubkey(&self) -> Pubkey { + self.tip_distribution_program_info.config_pda_and_bump.0 + } + + /// Given a bank, returns the current `tip_receiver` configured with the tip-payment program. + pub fn get_configured_tip_receiver(&self, bank: &Bank) -> Result { + Ok(self.get_tip_payment_config_account(bank)?.tip_receiver) + } + + pub fn get_tip_accounts(&self) -> HashSet { + HashSet::from([ + self.tip_payment_program_info.tip_pda_0.0, + self.tip_payment_program_info.tip_pda_1.0, + self.tip_payment_program_info.tip_pda_2.0, + self.tip_payment_program_info.tip_pda_3.0, + self.tip_payment_program_info.tip_pda_4.0, + self.tip_payment_program_info.tip_pda_5.0, + self.tip_payment_program_info.tip_pda_6.0, + self.tip_payment_program_info.tip_pda_7.0, + ]) + } + + pub fn get_tip_payment_config_account(&self, bank: &Bank) -> Result { + let config_data = bank + .get_account(&self.tip_payment_program_info.config_pda_bump.0) + .ok_or(TipError::AccountMissing( + self.tip_payment_program_info.config_pda_bump.0, + ))?; + + Ok(Config::try_deserialize(&mut config_data.data())?) + } + + /// Only called once during contract creation. + pub fn initialize_tip_payment_program_tx( + &self, + bank: &Bank, + keypair: &Keypair, + ) -> SanitizedTransaction { + let init_ix = Instruction { + program_id: self.tip_payment_program_info.program_id, + data: jito_tip_payment::instruction::Initialize { + _bumps: InitBumps { + config: self.tip_payment_program_info.config_pda_bump.1, + tip_payment_account_0: self.tip_payment_program_info.tip_pda_0.1, + tip_payment_account_1: self.tip_payment_program_info.tip_pda_1.1, + tip_payment_account_2: self.tip_payment_program_info.tip_pda_2.1, + tip_payment_account_3: self.tip_payment_program_info.tip_pda_3.1, + tip_payment_account_4: self.tip_payment_program_info.tip_pda_4.1, + tip_payment_account_5: self.tip_payment_program_info.tip_pda_5.1, + tip_payment_account_6: self.tip_payment_program_info.tip_pda_6.1, + tip_payment_account_7: self.tip_payment_program_info.tip_pda_7.1, + }, + } + .data(), + accounts: jito_tip_payment::accounts::Initialize { + config: self.tip_payment_program_info.config_pda_bump.0, + tip_payment_account_0: self.tip_payment_program_info.tip_pda_0.0, + tip_payment_account_1: self.tip_payment_program_info.tip_pda_1.0, + tip_payment_account_2: self.tip_payment_program_info.tip_pda_2.0, + tip_payment_account_3: self.tip_payment_program_info.tip_pda_3.0, + tip_payment_account_4: self.tip_payment_program_info.tip_pda_4.0, + tip_payment_account_5: self.tip_payment_program_info.tip_pda_5.0, + tip_payment_account_6: self.tip_payment_program_info.tip_pda_6.0, + tip_payment_account_7: self.tip_payment_program_info.tip_pda_7.0, + system_program: system_program::id(), + payer: keypair.pubkey(), + } + .to_account_metas(None), + }; + SanitizedTransaction::try_from_legacy_transaction( + Transaction::new_signed_with_payer( + &[init_ix], + Some(&keypair.pubkey()), + &[keypair], + bank.last_blockhash(), + ), + bank.get_reserved_account_keys(), + ) + .unwrap() + } + + /// Returns this validator's [TipDistributionAccount] PDA derived from the provided epoch. + pub fn get_my_tip_distribution_pda(&self, epoch: Epoch) -> Pubkey { + derive_tip_distribution_account_address( + &self.tip_distribution_program_info.program_id, + &self.tip_distribution_account_config.vote_account, + epoch, + ) + .0 + } + + /// Returns whether or not the tip-payment program should be initialized. + pub fn should_initialize_tip_payment_program(&self, bank: &Bank) -> bool { + match bank.get_account(&self.tip_payment_config_pubkey()) { + None => true, + Some(account) => account.owner() != &self.tip_payment_program_info.program_id, + } + } + + /// Returns whether or not the tip-distribution program's [Config] PDA should be initialized. + pub fn should_initialize_tip_distribution_config(&self, bank: &Bank) -> bool { + match bank.get_account(&self.tip_distribution_config_pubkey()) { + None => true, + Some(account) => account.owner() != &self.tip_distribution_program_info.program_id, + } + } + + /// Returns whether or not the current [TipDistributionAccount] PDA should be initialized for this epoch. + pub fn should_init_tip_distribution_account(&self, bank: &Bank) -> bool { + let pda = derive_tip_distribution_account_address( + &self.tip_distribution_program_info.program_id, + &self.tip_distribution_account_config.vote_account, + bank.epoch(), + ) + .0; + match bank.get_account(&pda) { + None => true, + // Since anyone can derive the PDA and send it lamports we must also check the owner is the program. + Some(account) => account.owner() != &self.tip_distribution_program_info.program_id, + } + } + + /// Creates an [Initialize] transaction object. + pub fn initialize_tip_distribution_config_tx( + &self, + bank: &Bank, + kp: &Keypair, + ) -> SanitizedTransaction { + let ix = initialize_ix( + self.tip_distribution_program_info.program_id, + InitializeArgs { + authority: kp.pubkey(), + expired_funds_account: kp.pubkey(), + num_epochs_valid: 10, + max_validator_commission_bps: 10_000, + bump: self.tip_distribution_program_info.config_pda_and_bump.1, + }, + InitializeAccounts { + config: self.tip_distribution_program_info.config_pda_and_bump.0, + system_program: system_program::id(), + initializer: kp.pubkey(), + }, + ); + + SanitizedTransaction::try_from_legacy_transaction( + Transaction::new_signed_with_payer( + &[ix], + Some(&kp.pubkey()), + &[kp], + bank.last_blockhash(), + ), + bank.get_reserved_account_keys(), + ) + .unwrap() + } + + /// Creates an [InitializeTipDistributionAccount] transaction object using the provided Epoch. + pub fn initialize_tip_distribution_account_tx( + &self, + bank: &Bank, + keypair: &Keypair, + ) -> SanitizedTransaction { + let (tip_distribution_account, bump) = derive_tip_distribution_account_address( + &self.tip_distribution_program_info.program_id, + &self.tip_distribution_account_config.vote_account, + bank.epoch(), + ); + + let ix = initialize_tip_distribution_account_ix( + self.tip_distribution_program_info.program_id, + InitializeTipDistributionAccountArgs { + merkle_root_upload_authority: self + .tip_distribution_account_config + .merkle_root_upload_authority, + validator_commission_bps: self.tip_distribution_account_config.commission_bps, + bump, + }, + InitializeTipDistributionAccountAccounts { + config: self.tip_distribution_program_info.config_pda_and_bump.0, + tip_distribution_account, + system_program: system_program::id(), + signer: keypair.pubkey(), + validator_vote_account: self.tip_distribution_account_config.vote_account, + }, + ); + + SanitizedTransaction::try_from_legacy_transaction( + Transaction::new_signed_with_payer( + &[ix], + Some(&keypair.pubkey()), + &[keypair], + bank.last_blockhash(), + ), + bank.get_reserved_account_keys(), + ) + .unwrap() + } + + /// Builds a transaction that changes the current tip receiver to new_tip_receiver. + /// The on-chain program will transfer tips sitting in the tip accounts to the tip receiver + /// before changing ownership. + pub fn change_tip_receiver_and_block_builder_tx( + &self, + new_tip_receiver: &Pubkey, + bank: &Bank, + keypair: &Keypair, + block_builder: &Pubkey, + block_builder_commission: u64, + ) -> Result { + let config = self.get_tip_payment_config_account(bank)?; + Ok(self.build_change_tip_receiver_and_block_builder_tx( + &config.tip_receiver, + new_tip_receiver, + bank, + keypair, + &config.block_builder, + block_builder, + block_builder_commission, + )) + } + + pub fn build_change_tip_receiver_and_block_builder_tx( + &self, + old_tip_receiver: &Pubkey, + new_tip_receiver: &Pubkey, + bank: &Bank, + keypair: &Keypair, + old_block_builder: &Pubkey, + block_builder: &Pubkey, + block_builder_commission: u64, + ) -> SanitizedTransaction { + let change_tip_ix = Instruction { + program_id: self.tip_payment_program_info.program_id, + data: jito_tip_payment::instruction::ChangeTipReceiver {}.data(), + accounts: jito_tip_payment::accounts::ChangeTipReceiver { + config: self.tip_payment_program_info.config_pda_bump.0, + old_tip_receiver: *old_tip_receiver, + new_tip_receiver: *new_tip_receiver, + block_builder: *old_block_builder, + tip_payment_account_0: self.tip_payment_program_info.tip_pda_0.0, + tip_payment_account_1: self.tip_payment_program_info.tip_pda_1.0, + tip_payment_account_2: self.tip_payment_program_info.tip_pda_2.0, + tip_payment_account_3: self.tip_payment_program_info.tip_pda_3.0, + tip_payment_account_4: self.tip_payment_program_info.tip_pda_4.0, + tip_payment_account_5: self.tip_payment_program_info.tip_pda_5.0, + tip_payment_account_6: self.tip_payment_program_info.tip_pda_6.0, + tip_payment_account_7: self.tip_payment_program_info.tip_pda_7.0, + signer: keypair.pubkey(), + } + .to_account_metas(None), + }; + let change_block_builder_ix = Instruction { + program_id: self.tip_payment_program_info.program_id, + data: jito_tip_payment::instruction::ChangeBlockBuilder { + block_builder_commission, + } + .data(), + accounts: jito_tip_payment::accounts::ChangeBlockBuilder { + config: self.tip_payment_program_info.config_pda_bump.0, + tip_receiver: *new_tip_receiver, // tip receiver will have just changed in previous ix + old_block_builder: *old_block_builder, + new_block_builder: *block_builder, + tip_payment_account_0: self.tip_payment_program_info.tip_pda_0.0, + tip_payment_account_1: self.tip_payment_program_info.tip_pda_1.0, + tip_payment_account_2: self.tip_payment_program_info.tip_pda_2.0, + tip_payment_account_3: self.tip_payment_program_info.tip_pda_3.0, + tip_payment_account_4: self.tip_payment_program_info.tip_pda_4.0, + tip_payment_account_5: self.tip_payment_program_info.tip_pda_5.0, + tip_payment_account_6: self.tip_payment_program_info.tip_pda_6.0, + tip_payment_account_7: self.tip_payment_program_info.tip_pda_7.0, + signer: keypair.pubkey(), + } + .to_account_metas(None), + }; + SanitizedTransaction::try_from_legacy_transaction( + Transaction::new_signed_with_payer( + &[change_tip_ix, change_block_builder_ix], + Some(&keypair.pubkey()), + &[keypair], + bank.last_blockhash(), + ), + bank.get_reserved_account_keys(), + ) + .unwrap() + } + + /// Returns the balance of all the MEV tip accounts + pub fn get_tip_account_balances(&self, bank: &Arc) -> Vec<(Pubkey, u64)> { + let accounts = self.get_tip_accounts(); + accounts + .into_iter() + .map(|account| { + let balance = bank.get_balance(&account); + (account, balance) + }) + .collect() + } + + /// Returns the balance of all the MEV tip accounts above the rent-exempt amount. + /// NOTE: the on-chain program has rent_exempt = force + pub fn get_tip_account_balances_above_rent_exempt( + &self, + bank: &Arc, + ) -> Vec<(Pubkey, u64)> { + let accounts = self.get_tip_accounts(); + accounts + .into_iter() + .map(|account| { + let account_data = bank.get_account(&account).unwrap_or_default(); + let balance = bank.get_balance(&account); + let rent_exempt = + bank.get_minimum_balance_for_rent_exemption(account_data.data().len()); + // NOTE: don't unwrap here in case bug in on-chain program, don't want all validators to crash + // if program gets stuck in bad state + (account, balance.checked_sub(rent_exempt).unwrap_or_else(|| { + warn!("balance is below rent exempt amount. balance: {} rent_exempt: {} acc size: {}", balance, rent_exempt, TipPaymentAccount::SIZE); + 0 + })) + }) + .collect() + } + + /// Return a bundle that is capable of calling the initialize instructions on the two tip payment programs + /// This is mainly helpful for local development and shouldn't run on testnet and mainnet, assuming the + /// correct TipManager configuration is set. + pub fn get_initialize_tip_programs_bundle( + &self, + bank: &Bank, + keypair: &Keypair, + ) -> Option { + let maybe_init_tip_payment_config_tx = if self.should_initialize_tip_payment_program(bank) { + debug!("should_initialize_tip_payment_program=true"); + Some(self.initialize_tip_payment_program_tx(bank, keypair)) + } else { + None + }; + + let maybe_init_tip_distro_config_tx = + if self.should_initialize_tip_distribution_config(bank) { + debug!("should_initialize_tip_distribution_config=true"); + Some(self.initialize_tip_distribution_config_tx(bank, keypair)) + } else { + None + }; + + let transactions = [ + maybe_init_tip_payment_config_tx, + maybe_init_tip_distro_config_tx, + ] + .into_iter() + .flatten() + .collect::>(); + + if transactions.is_empty() { + None + } else { + let bundle_id = derive_bundle_id_from_sanitized_transactions(&transactions); + Some(SanitizedBundle { + transactions, + bundle_id, + }) + } + } + + pub fn get_tip_programs_crank_bundle( + &self, + bank: &Bank, + keypair: &Keypair, + block_builder_fee_info: &BlockBuilderFeeInfo, + ) -> Result> { + let maybe_init_tip_distro_account_tx = if self.should_init_tip_distribution_account(bank) { + debug!("should_init_tip_distribution_account=true"); + Some(self.initialize_tip_distribution_account_tx(bank, keypair)) + } else { + None + }; + + let configured_tip_receiver = self.get_configured_tip_receiver(bank)?; + let my_tip_receiver = self.get_my_tip_distribution_pda(bank.epoch()); + let maybe_change_tip_receiver_tx = if configured_tip_receiver != my_tip_receiver { + debug!("change_tip_receiver=true"); + Some(self.change_tip_receiver_and_block_builder_tx( + &my_tip_receiver, + bank, + keypair, + &block_builder_fee_info.block_builder, + block_builder_fee_info.block_builder_commission, + )?) + } else { + None + }; + debug!( + "maybe_change_tip_receiver_tx: {:?}", + maybe_change_tip_receiver_tx + ); + + let transactions = [ + maybe_init_tip_distro_account_tx, + maybe_change_tip_receiver_tx, + ] + .into_iter() + .flatten() + .collect::>(); + + if transactions.is_empty() { + Ok(None) + } else { + let bundle_id = derive_bundle_id_from_sanitized_transactions(&transactions); + Ok(Some(SanitizedBundle { + transactions, + bundle_id, + })) + } + } +} diff --git a/core/src/tpu.rs b/core/src/tpu.rs index c76d2dd4d0..367223f992 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -6,14 +6,21 @@ use { crate::{ banking_stage::BankingStage, banking_trace::{BankingTracer, TracerThread}, + bundle_stage::{bundle_account_locker::BundleAccountLocker, BundleStage}, cluster_info_vote_listener::{ ClusterInfoVoteListener, DuplicateConfirmedSlotsSender, GossipVerifiedVoteHashSender, VerifiedVoteSender, VoteTracker, }, fetch_stage::FetchStage, + proxy::{ + block_engine_stage::{BlockBuilderFeeInfo, BlockEngineConfig, BlockEngineStage}, + fetch_stage_manager::FetchStageManager, + relayer_stage::{RelayerConfig, RelayerStage}, + }, sigverify::TransactionSigVerifier, sigverify_stage::SigVerifyStage, staked_nodes_updater_service::StakedNodesUpdaterService, + tip_manager::{TipManager, TipManagerConfig}, tpu_entry_notifier::TpuEntryNotifier, validator::{BlockProductionMethod, GeneratorConfig}, }, @@ -35,7 +42,12 @@ use { prioritization_fee_cache::PrioritizationFeeCache, vote_sender_types::{ReplayVoteReceiver, ReplayVoteSender}, }, - solana_sdk::{clock::Slot, pubkey::Pubkey, quic::NotifyKeyUpdate, signature::Keypair}, + solana_sdk::{ + clock::Slot, + pubkey::Pubkey, + quic::NotifyKeyUpdate, + signature::{Keypair, Signer}, + }, solana_streamer::{ nonblocking::quic::{DEFAULT_MAX_STREAMS_PER_MS, DEFAULT_WAIT_FOR_CHUNK_TIMEOUT}, quic::{ @@ -45,9 +57,9 @@ use { }, solana_turbine::broadcast_stage::{BroadcastStage, BroadcastStageType}, std::{ - collections::HashMap, + collections::{HashMap, HashSet}, net::{SocketAddr, UdpSocket}, - sync::{atomic::AtomicBool, Arc, RwLock}, + sync::{atomic::AtomicBool, Arc, Mutex, RwLock}, thread, time::Duration, }, @@ -78,6 +90,10 @@ pub struct Tpu { tpu_entry_notifier: Option, staked_nodes_updater_service: StakedNodesUpdaterService, tracer_thread_hdl: TracerThread, + relayer_stage: RelayerStage, + block_engine_stage: BlockEngineStage, + fetch_stage_manager: FetchStageManager, + bundle_stage: BundleStage, } impl Tpu { @@ -118,6 +134,11 @@ impl Tpu { block_production_method: BlockProductionMethod, enable_block_production_forwarding: bool, _generator_config: Option, /* vestigial code for replay invalidator */ + block_engine_config: Arc>, + relayer_config: Arc>, + tip_manager_config: TipManagerConfig, + shred_receiver_address: Arc>>, + preallocated_bundle_cost: u64, ) -> (Self, Vec>) { let TpuSockets { transactions: transactions_sockets, @@ -128,7 +149,10 @@ impl Tpu { transactions_forwards_quic: transactions_forwards_quic_sockets, } = sockets; - let (packet_sender, packet_receiver) = unbounded(); + // Packets from fetch stage and quic server are intercepted and sent through fetch_stage_manager + // If relayer is connected, packets are dropped. If not, packets are forwarded on to packet_sender + let (packet_intercept_sender, packet_intercept_receiver) = unbounded(); + let (vote_packet_sender, vote_packet_receiver) = unbounded(); let (forwarded_packet_sender, forwarded_packet_receiver) = unbounded(); let fetch_stage = FetchStage::new_with_sender( @@ -136,7 +160,7 @@ impl Tpu { tpu_forwards_sockets, tpu_vote_sockets, exit.clone(), - &packet_sender, + &packet_intercept_sender, &vote_packet_sender, &forwarded_packet_sender, forwarded_packet_receiver, @@ -164,7 +188,7 @@ impl Tpu { "quic_streamer_tpu", transactions_quic_sockets, keypair, - packet_sender, + packet_intercept_sender, exit.clone(), MAX_QUIC_CONNECTIONS_PER_PEER, staked_nodes.clone(), @@ -199,8 +223,10 @@ impl Tpu { ) .unwrap(); + let (packet_sender, packet_receiver) = unbounded(); + let sigverify_stage = { - let verifier = TransactionSigVerifier::new(non_vote_sender); + let verifier = TransactionSigVerifier::new(non_vote_sender.clone()); SigVerifyStage::new(packet_receiver, verifier, "solSigVerTpu", "tpu-verifier") }; @@ -218,6 +244,41 @@ impl Tpu { let (gossip_vote_sender, gossip_vote_receiver) = banking_tracer.create_channel_gossip_vote(); + + let block_builder_fee_info = Arc::new(Mutex::new(BlockBuilderFeeInfo { + block_builder: cluster_info.keypair().pubkey(), + block_builder_commission: 0, + })); + + let (bundle_sender, bundle_receiver) = unbounded(); + let block_engine_stage = BlockEngineStage::new( + block_engine_config, + bundle_sender, + cluster_info.clone(), + packet_sender.clone(), + non_vote_sender.clone(), + exit.clone(), + &block_builder_fee_info, + ); + + let (heartbeat_tx, heartbeat_rx) = unbounded(); + let fetch_stage_manager = FetchStageManager::new( + cluster_info.clone(), + heartbeat_rx, + packet_intercept_receiver, + packet_sender.clone(), + exit.clone(), + ); + + let relayer_stage = RelayerStage::new( + relayer_config, + cluster_info.clone(), + heartbeat_tx, + packet_sender, + non_vote_sender, + exit.clone(), + ); + let cluster_info_vote_listener = ClusterInfoVoteListener::new( exit.clone(), cluster_info.clone(), @@ -234,6 +295,15 @@ impl Tpu { duplicate_confirmed_slot_sender, ); + let tip_manager = TipManager::new(tip_manager_config); + + let bundle_account_locker = BundleAccountLocker::default(); + + // tip accounts can't be used in BankingStage to avoid someone from stealing tips mid-slot. + // it also helps reduce surface area for potential account contention + let mut blacklisted_accounts = HashSet::new(); + blacklisted_accounts.insert(tip_manager.tip_payment_config_pubkey()); + blacklisted_accounts.extend(tip_manager.get_tip_accounts()); let banking_stage = BankingStage::new( block_production_method, cluster_info, @@ -241,13 +311,31 @@ impl Tpu { non_vote_receiver, tpu_vote_receiver, gossip_vote_receiver, - transaction_status_sender, - replay_vote_sender, + transaction_status_sender.clone(), + replay_vote_sender.clone(), log_messages_bytes_limit, connection_cache.clone(), bank_forks.clone(), prioritization_fee_cache, enable_block_production_forwarding, + blacklisted_accounts, + bundle_account_locker.clone(), + ); + + let bundle_stage = BundleStage::new( + cluster_info, + poh_recorder, + bundle_receiver, + transaction_status_sender, + replay_vote_sender, + log_messages_bytes_limit, + exit.clone(), + tip_manager, + bundle_account_locker, + &block_builder_fee_info, + preallocated_bundle_cost, + bank_forks.clone(), + prioritization_fee_cache, ); let (entry_receiver, tpu_entry_notifier) = @@ -274,6 +362,7 @@ impl Tpu { bank_forks, shred_version, turbine_quic_endpoint_sender, + shred_receiver_address, ); ( @@ -289,6 +378,10 @@ impl Tpu { tpu_entry_notifier, staked_nodes_updater_service, tracer_thread_hdl, + block_engine_stage, + relayer_stage, + fetch_stage_manager, + bundle_stage, }, vec![key_updater, forwards_key_updater], ) @@ -304,6 +397,10 @@ impl Tpu { self.staked_nodes_updater_service.join(), self.tpu_quic_t.join(), self.tpu_forwards_quic_t.join(), + self.bundle_stage.join(), + self.relayer_stage.join(), + self.block_engine_stage.join(), + self.fetch_stage_manager.join(), ]; let broadcast_result = self.broadcast_stage.join(); for result in results { diff --git a/core/src/tpu_entry_notifier.rs b/core/src/tpu_entry_notifier.rs index 22994455e8..e226b0ef1f 100644 --- a/core/src/tpu_entry_notifier.rs +++ b/core/src/tpu_entry_notifier.rs @@ -61,43 +61,57 @@ impl TpuEntryNotifier { current_index: &mut usize, current_transaction_index: &mut usize, ) -> Result<(), RecvTimeoutError> { - let (bank, (entry, tick_height)) = entry_receiver.recv_timeout(Duration::from_secs(1))?; + let WorkingBankEntry { + bank, + entries_ticks, + } = entry_receiver.recv_timeout(Duration::from_secs(1))?; let slot = bank.slot(); - let index = if slot != *current_slot { - *current_index = 0; - *current_transaction_index = 0; - *current_slot = slot; - 0 - } else { - *current_index += 1; - *current_index - }; + let mut indices_sent = vec![]; - let entry_summary = EntrySummary { - num_hashes: entry.num_hashes, - hash: entry.hash, - num_transactions: entry.transactions.len() as u64, - }; - if let Err(err) = entry_notification_sender.send(EntryNotification { - slot, - index, - entry: entry_summary, - starting_transaction_index: *current_transaction_index, - }) { - warn!( + entries_ticks.iter().for_each(|(entry, _)| { + let index = if slot != *current_slot { + *current_index = 0; + *current_transaction_index = 0; + *current_slot = slot; + 0 + } else { + *current_index += 1; + *current_index + }; + + let entry_summary = EntrySummary { + num_hashes: entry.num_hashes, + hash: entry.hash, + num_transactions: entry.transactions.len() as u64, + }; + if let Err(err) = entry_notification_sender.send(EntryNotification { + slot, + index, + entry: entry_summary, + starting_transaction_index: *current_transaction_index + }) { + warn!( "Failed to send slot {slot:?} entry {index:?} from Tpu to EntryNotifierService, error {err:?}", ); - } - *current_transaction_index += entry.transactions.len(); + } - if let Err(err) = broadcast_entry_sender.send((bank, (entry, tick_height))) { + *current_transaction_index += entry.transactions.len(); + + indices_sent.push(index); + }); + + if let Err(err) = broadcast_entry_sender.send(WorkingBankEntry { + bank, + entries_ticks, + }) { warn!( - "Failed to send slot {slot:?} entry {index:?} from Tpu to BroadcastStage, error {err:?}", + "Failed to send slot {slot:?} entries {indices_sent:?} from Tpu to BroadcastStage, error {err:?}", ); // If the BroadcastStage channel is closed, the validator has halted. Try to exit // gracefully. exit.store(true, Ordering::Relaxed); } + Ok(()) } diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 4dcd7bbfa3..9e39bf390e 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -159,6 +159,7 @@ impl Tvu { outstanding_repair_requests: Arc>, cluster_slots: Arc, wen_restart_repair_slots: Option>>>, + shred_receiver_addr: Arc>>, ) -> Result { let TvuSockets { repair: repair_socket, @@ -207,6 +208,7 @@ impl Tvu { retransmit_receiver, max_slots.clone(), Some(rpc_subscriptions.clone()), + shred_receiver_addr, ); let (ancestor_duplicate_slots_sender, ancestor_duplicate_slots_receiver) = unbounded(); @@ -521,6 +523,7 @@ pub mod tests { outstanding_repair_requests, cluster_slots, None, + Arc::new(RwLock::new(None)), ) .expect("assume success"); exit.store(true, Ordering::Relaxed); diff --git a/core/src/validator.rs b/core/src/validator.rs index c720759a69..d4a8cb84e2 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -15,6 +15,7 @@ use { ExternalRootSource, Tower, }, poh_timing_report_service::PohTimingReportService, + proxy::{block_engine_stage::BlockEngineConfig, relayer_stage::RelayerConfig}, repair::{self, serve_repair::ServeRepair, serve_repair_service::ServeRepairService}, rewards_recorder_service::{RewardsRecorderSender, RewardsRecorderService}, sample_performance_service::SamplePerformanceService, @@ -24,6 +25,7 @@ use { system_monitor_service::{ verify_net_stats_access, SystemMonitorService, SystemMonitorStatsReportConfig, }, + tip_manager::TipManagerConfig, tpu::{Tpu, TpuSockets, DEFAULT_TPU_COALESCE}, tvu::{Tvu, TvuConfig, TvuSockets}, }, @@ -105,6 +107,10 @@ use { snapshot_hash::StartingSnapshotHashes, snapshot_utils::{self, clean_orphaned_account_snapshot_dirs}, }, + solana_runtime_plugin::{ + runtime_plugin_admin_rpc_service::RuntimePluginManagerRpcRequest, + runtime_plugin_service::RuntimePluginService, + }, solana_sdk::{ clock::Slot, epoch_schedule::MAX_LEADER_SCHEDULE_EPOCH_OFFSET, @@ -129,7 +135,7 @@ use { path::{Path, PathBuf}, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, - Arc, RwLock, + Arc, Mutex, RwLock, }, thread::{sleep, Builder, JoinHandle}, time::{Duration, Instant}, @@ -215,7 +221,8 @@ pub struct ValidatorConfig { pub rpc_config: JsonRpcConfig, /// Specifies which plugins to start up with pub on_start_geyser_plugin_config_files: Option>, - pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub) + pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, + // (JsonRpc, JsonRpcPubSub) pub pubsub_config: PubSubConfig, pub snapshot_config: SnapshotConfig, pub max_ledger_shreds: Option, @@ -225,10 +232,14 @@ pub struct ValidatorConfig { pub fixed_leader_schedule: Option, pub wait_for_supermajority: Option, pub new_hard_forks: Option>, - pub known_validators: Option>, // None = trust all - pub repair_validators: Option>, // None = repair from all - pub repair_whitelist: Arc>>, // Empty = repair with all - pub gossip_validators: Option>, // None = gossip with all + pub known_validators: Option>, + // None = trust all + pub repair_validators: Option>, + // None = repair from all + pub repair_whitelist: Arc>>, + // Empty = repair with all + pub gossip_validators: Option>, + // None = gossip with all pub accounts_hash_interval_slots: u64, pub max_genesis_archive_unpacked_size: u64, pub wal_recovery_mode: Option, @@ -275,6 +286,12 @@ pub struct ValidatorConfig { pub replay_forks_threads: NonZeroUsize, pub replay_transactions_threads: NonZeroUsize, pub delay_leader_block_for_pending_fork: bool, + pub relayer_config: Arc>, + pub block_engine_config: Arc>, + // Using Option inside RwLock is ugly, but only convenient way to allow toggle on/off + pub shred_receiver_address: Arc>>, + pub tip_manager_config: TipManagerConfig, + pub preallocated_bundle_cost: u64, } impl Default for ValidatorConfig { @@ -347,6 +364,11 @@ impl Default for ValidatorConfig { replay_forks_threads: NonZeroUsize::new(1).expect("1 is non-zero"), replay_transactions_threads: NonZeroUsize::new(1).expect("1 is non-zero"), delay_leader_block_for_pending_fork: false, + relayer_config: Arc::new(Mutex::new(RelayerConfig::default())), + block_engine_config: Arc::new(Mutex::new(BlockEngineConfig::default())), + shred_receiver_address: Arc::new(RwLock::new(None)), + tip_manager_config: TipManagerConfig::default(), + preallocated_bundle_cost: u64::default(), } } } @@ -385,7 +407,8 @@ impl ValidatorConfig { // having to watch log messages. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] pub enum ValidatorStartProgress { - Initializing, // Catch all, default state + Initializing, + // Catch all, default state SearchingForRpcService, DownloadingSnapshot { slot: Slot, @@ -399,7 +422,8 @@ pub enum ValidatorStartProgress { max_slot: Slot, }, StartingServices, - Halted, // Validator halted due to `--dev-halt-at-slot` argument + Halted, + // Validator halted due to `--dev-halt-at-slot` argument WaitingForSupermajority { slot: Slot, gossip_stake_percent: u64, @@ -516,6 +540,10 @@ impl Validator { tpu_enable_udp: bool, tpu_max_connections_per_ipaddr_per_minute: u64, admin_rpc_service_post_init: Arc>>, + runtime_plugin_configs_and_request_rx: Option<( + Vec, + Receiver, + )>, ) -> Result { let start_time = Instant::now(); @@ -929,6 +957,17 @@ impl Validator { None, )); + if let Some((runtime_plugin_configs, request_rx)) = runtime_plugin_configs_and_request_rx { + RuntimePluginService::start( + &runtime_plugin_configs, + request_rx, + bank_forks.clone(), + block_commitment_cache.clone(), + exit.clone(), + ) + .map_err(|e| format!("Failed to start runtime plugin service: {e:?}"))?; + } + let max_slots = Arc::new(MaxSlots::default()); let startup_verification_complete; @@ -1369,6 +1408,7 @@ impl Validator { outstanding_repair_requests.clone(), cluster_slots.clone(), wen_restart_repair_slots.clone(), + config.shred_receiver_address.clone(), )?; if in_wen_restart { @@ -1437,6 +1477,11 @@ impl Validator { config.block_production_method.clone(), config.enable_block_production_forwarding, config.generator_config.clone(), + config.block_engine_config.clone(), + config.relayer_config.clone(), + config.tip_manager_config.clone(), + config.shred_receiver_address.clone(), + config.preallocated_bundle_cost, ); datapoint_info!( @@ -1461,6 +1506,9 @@ impl Validator { repair_socket: Arc::new(node.sockets.repair), outstanding_repair_requests, cluster_slots, + block_engine_config: config.block_engine_config.clone(), + relayer_config: config.relayer_config.clone(), + shred_receiver_address: config.shred_receiver_address.clone(), }); Ok(Self { @@ -1928,6 +1976,7 @@ fn load_blockstore( .map(|service| service.sender()), accounts_update_notifier, exit, + true, ) .map_err(|err| err.to_string())?; @@ -2619,6 +2668,7 @@ mod tests { DEFAULT_TPU_ENABLE_UDP, 32, // max connections per IpAddr per minute for test Arc::new(RwLock::new(None)), + None, ) .expect("assume successful validator start"); assert_eq!( @@ -2696,7 +2746,7 @@ mod tests { Arc::new(RwLock::new(vec![Arc::new(vote_account_keypair)])), vec![leader_node.info.clone()], &config, - true, // should_check_duplicate_instance. + true, // should_check_duplicate_instance None, // rpc_to_plugin_manager_receiver Arc::new(RwLock::new(ValidatorStartProgress::default())), SocketAddrSpace::Unspecified, @@ -2705,6 +2755,7 @@ mod tests { DEFAULT_TPU_ENABLE_UDP, 32, // max connections per IpAddr per minute for test Arc::new(RwLock::new(None)), + None, ) .expect("assume successful validator start") }) @@ -2821,7 +2872,7 @@ mod tests { assert!(is_snapshot_config_valid( &new_snapshot_config(300, 200), - 100 + 100, )); let default_accounts_hash_interval = @@ -2829,62 +2880,62 @@ mod tests { assert!(is_snapshot_config_valid( &new_snapshot_config( snapshot_bank_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, - snapshot_bank_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS + snapshot_bank_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, ), default_accounts_hash_interval, )); assert!(is_snapshot_config_valid( &new_snapshot_config( snapshot_bank_utils::DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, - DISABLED_SNAPSHOT_ARCHIVE_INTERVAL + DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, ), - default_accounts_hash_interval + default_accounts_hash_interval, )); assert!(is_snapshot_config_valid( &new_snapshot_config( snapshot_bank_utils::DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, - DISABLED_SNAPSHOT_ARCHIVE_INTERVAL + DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, ), - default_accounts_hash_interval + default_accounts_hash_interval, )); assert!(is_snapshot_config_valid( &new_snapshot_config( DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, - DISABLED_SNAPSHOT_ARCHIVE_INTERVAL + DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, ), - Slot::MAX + Slot::MAX, )); assert!(!is_snapshot_config_valid(&new_snapshot_config(0, 100), 100)); assert!(!is_snapshot_config_valid(&new_snapshot_config(100, 0), 100)); assert!(!is_snapshot_config_valid( &new_snapshot_config(42, 100), - 100 + 100, )); assert!(!is_snapshot_config_valid( &new_snapshot_config(100, 42), - 100 + 100, )); assert!(!is_snapshot_config_valid( &new_snapshot_config(100, 100), - 100 + 100, )); assert!(!is_snapshot_config_valid( &new_snapshot_config(100, 200), - 100 + 100, )); assert!(!is_snapshot_config_valid( &new_snapshot_config(444, 200), - 100 + 100, )); assert!(!is_snapshot_config_valid( &new_snapshot_config(400, 222), - 100 + 100, )); assert!(is_snapshot_config_valid( &SnapshotConfig::new_load_only(), - 100 + 100, )); assert!(is_snapshot_config_valid( &SnapshotConfig { @@ -2892,7 +2943,7 @@ mod tests { incremental_snapshot_archive_interval_slots: 37, ..SnapshotConfig::new_load_only() }, - 100 + 100, )); assert!(is_snapshot_config_valid( &SnapshotConfig { @@ -2900,7 +2951,7 @@ mod tests { incremental_snapshot_archive_interval_slots: DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, ..SnapshotConfig::new_load_only() }, - 100 + 100, )); } diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index 7ec41c673f..c516f195f3 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -439,6 +439,7 @@ fn test_snapshots_have_expected_epoch_accounts_hash() { if let Some(full_snapshot_archive_info) = snapshot_utils::get_highest_full_snapshot_archive_info( &snapshot_config.full_snapshot_archives_dir, + None, ) { if full_snapshot_archive_info.slot() == bank.slot() { @@ -562,6 +563,7 @@ fn test_background_services_request_handling_for_epoch_accounts_hash() { info!("Taking full snapshot..."); while snapshot_utils::get_highest_full_snapshot_archive_slot( &snapshot_config.full_snapshot_archives_dir, + None, ) != Some(bank.slot()) { trace!("waiting for full snapshot..."); diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 4268e48260..ec302c4ca0 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -780,6 +780,7 @@ fn test_snapshots_with_background_services( &snapshot_test_config .snapshot_config .full_snapshot_archives_dir, + None, ) != Some(slot) { assert!( @@ -798,6 +799,7 @@ fn test_snapshots_with_background_services( .snapshot_config .incremental_snapshot_archives_dir, last_full_snapshot_slot.unwrap(), + None, ) != Some(slot) { assert!( diff --git a/cost-model/src/cost_tracker.rs b/cost-model/src/cost_tracker.rs index 0c731f946e..e0487ace86 100644 --- a/cost-model/src/cost_tracker.rs +++ b/cost-model/src/cost_tracker.rs @@ -130,6 +130,10 @@ impl CostTracker { self.vote_cost_limit = vote_cost_limit; } + pub fn set_block_cost_limit(&mut self, block_cost_limit: u64) { + self.block_cost_limit = block_cost_limit; + } + pub fn in_flight_transaction_count(&self) -> usize { self.in_flight_transaction_count } @@ -192,6 +196,10 @@ impl CostTracker { self.block_cost } + pub fn block_cost_limit(&self) -> u64 { + self.block_cost_limit + } + pub fn transaction_count(&self) -> u64 { self.transaction_count } diff --git a/deploy_programs b/deploy_programs new file mode 100755 index 0000000000..cbdf837e92 --- /dev/null +++ b/deploy_programs @@ -0,0 +1,17 @@ +#!/usr/bin/env sh +# Deploys the tip payment and tip distribution programs on local validator at predetermined address +set -eux + +WALLET_LOCATION=~/.config/solana/id.json + +# build this solana binary to ensure we're using a version compatible with the validator +cargo b --release --bin solana + +./target/release/solana airdrop -ul 1000 $WALLET_LOCATION + +(cd jito-programs/tip-payment && anchor build) + +# NOTE: make sure the declare_id! is set correctly in the programs +# Also, || true to make sure if fails the first time around, tip_payment can still be deployed +RUST_INFO=trace ./target/release/solana deploy --keypair $WALLET_LOCATION -ul ./jito-programs/tip-payment/target/deploy/tip_distribution.so ./jito-programs/tip-payment/dev/dev_tip_distribution.json || true +RUST_INFO=trace ./target/release/solana deploy --keypair $WALLET_LOCATION -ul ./jito-programs/tip-payment/target/deploy/tip_payment.so ./jito-programs/tip-payment/dev/dev_tip_payment.json diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 0000000000..bab9a1c02f --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,48 @@ +FROM rust:1.64-slim-bullseye as builder + +# Add Google Protocol Buffers for Libra's metrics library. +ENV PROTOC_VERSION 3.8.0 +ENV PROTOC_ZIP protoc-$PROTOC_VERSION-linux-x86_64.zip + +RUN set -x \ + && apt update \ + && apt install -y \ + clang \ + cmake \ + libudev-dev \ + make \ + unzip \ + libssl-dev \ + pkg-config \ + zlib1g-dev \ + curl \ + && rustup component add rustfmt \ + && rustup component add clippy \ + && rustc --version \ + && cargo --version \ + && curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \ + && unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \ + && unzip -o $PROTOC_ZIP -d /usr/local include/* \ + && rm -f $PROTOC_ZIP + + +WORKDIR /solana +COPY . . +RUN mkdir -p docker-output + +ARG ci_commit +# NOTE: Keep this here before build since variable is referenced during CI build step. +ENV CI_COMMIT=$ci_commit + +ARG debug + +# Uses docker buildkit to cache the image. +# /usr/local/cargo/git needed for crossbeam patch +RUN --mount=type=cache,mode=0777,target=/solana/target \ + --mount=type=cache,mode=0777,target=/usr/local/cargo/registry \ + --mount=type=cache,mode=0777,target=/usr/local/cargo/git \ + if [ "$debug" = "false" ] ; then \ + ./cargo stable build --release && cp target/release/solana* ./docker-output; \ + else \ + RUSTFLAGS='-g -C force-frame-pointers=yes' ./cargo stable build --release && cp target/release/solana* ./docker-output; \ + fi diff --git a/docs/src/cli/install.md b/docs/src/cli/install.md index c9a5c682d4..d4be9638a4 100644 --- a/docs/src/cli/install.md +++ b/docs/src/cli/install.md @@ -20,11 +20,11 @@ on your preferred workflow: - Open your favorite Terminal application - Install the Agave release - [LATEST_AGAVE_RELEASE_VERSION](https://github.com/anza-xyz/agave/releases/tag/LATEST_AGAVE_RELEASE_VERSION) + [LATEST_AGAVE_RELEASE_VERSION](https://github.com/jito-foundation/jito-solana/releases/tag/LATEST_AGAVE_RELEASE_VERSION) on your machine by running: ```bash -sh -c "$(curl -sSfL https://release.anza.xyz/LATEST_AGAVE_RELEASE_VERSION/install)" +sh -c "$(curl -sSfL https://release.jito.wtf/LATEST_AGAVE_RELEASE_VERSION/install)" ``` - You can replace `LATEST_AGAVE_RELEASE_VERSION` with the release tag matching @@ -38,7 +38,7 @@ downloading LATEST_AGAVE_RELEASE_VERSION installer Configuration: /home/solana/.config/solana/install/config.yml Active release directory: /home/solana/.local/share/solana/install/active_release * Release version: LATEST_AGAVE_RELEASE_VERSION -* Release URL: https://github.com/anza-xyz/agave/releases/download/LATEST_AGAVE_RELEASE_VERSION/solana-release-x86_64-unknown-linux-gnu.tar.bz2 +* Release URL: https://github.com/jito-foundation/jito-solana/releases/download/LATEST_AGAVE_RELEASE_VERSION/solana-release-x86_64-unknown-linux-gnu.tar.bz2 Update successful ``` @@ -65,16 +65,16 @@ solana --version - Open a Command Prompt (`cmd.exe`) as an Administrator - - Search for Command Prompt in the Windows search bar. When the Command Prompt - app appears, right-click and select “Open as Administrator”. If you are - prompted by a pop-up window asking “Do you want to allow this app to make - changes to your device?”, click Yes. + - Search for Command Prompt in the Windows search bar. When the Command Prompt + app appears, right-click and select “Open as Administrator”. If you are + prompted by a pop-up window asking “Do you want to allow this app to make + changes to your device?”, click Yes. - Copy and paste the following command, then press Enter to download the Solana installer into a temporary directory: ```bash -cmd /c "curl https://release.anza.xyz/LATEST_AGAVE_RELEASE_VERSION/agave-install-init-x86_64-pc-windows-msvc.exe --output C:\agave-install-tmp\agave-install-init.exe --create-dirs" +cmd /c "curl https://release.jito.wtf/LATEST_AGAVE_RELEASE_VERSION/agave-install-init-x86_64-pc-windows-msvc.exe --output C:\agave-install-tmp\agave-install-init.exe --create-dirs" ``` - Copy and paste the following command, then press Enter to install the latest @@ -89,8 +89,8 @@ C:\agave-install-tmp\agave-install-init.exe LATEST_AGAVE_RELEASE_VERSION - Close the command prompt window and re-open a new command prompt window as a normal user - - Search for "Command Prompt" in the search bar, then left click on the - Command Prompt app icon, no need to run as Administrator) + - Search for "Command Prompt" in the search bar, then left click on the + Command Prompt app icon, no need to run as Administrator) - Confirm you have the desired version of `solana` installed by entering: ```bash @@ -108,9 +108,7 @@ manually download and install the binaries. ### Linux Download the binaries by navigating to -[https://github.com/anza-xyz/agave/releases/latest](https://github.com/anza-xyz/agave/releases/latest), -download **solana-release-x86_64-unknown-linux-gnu.tar.bz2**, then extract the -archive: +[https://github.com/jito-foundation/jito-solana/releases/latest](https://github.com/jito-foundation/jito-solana/releases/latest), ```bash tar jxf solana-release-x86_64-unknown-linux-gnu.tar.bz2 @@ -121,9 +119,7 @@ export PATH=$PWD/bin:$PATH ### MacOS Download the binaries by navigating to -[https://github.com/anza-xyz/agave/releases/latest](https://github.com/anza-xyz/agave/releases/latest), -download **solana-release-x86_64-apple-darwin.tar.bz2**, then extract the -archive: +[https://github.com/jito-foundation/jito-solana/releases/latest](https://github.com/jito-foundation/jito-solana/releases/latest), ```bash tar jxf solana-release-x86_64-apple-darwin.tar.bz2 @@ -134,10 +130,7 @@ export PATH=$PWD/bin:$PATH ### Windows - Download the binaries by navigating to - [https://github.com/anza-xyz/agave/releases/latest](https://github.com/anza-xyz/agave/releases/latest), - download **solana-release-x86_64-pc-windows-msvc.tar.bz2**, then extract the - archive using WinZip or similar. - + [https://github.com/jito-foundation/jito-solana/releases/latest](https://github.com/jito-foundation/jito-solana/releases/latest), - Open a Command Prompt and navigate to the directory into which you extracted the binaries and run: @@ -242,9 +235,7 @@ above. After installing the prerequisites, proceed with building Solana from source, navigate to -[Solana's GitHub releases page](https://github.com/anza-xyz/agave/releases/latest), -and download the **Source Code** archive. Extract the code and build the -binaries with: +[Solana's GitHub releases page](https://github.com/jito-foundation/jito-solana/releases/latest), ```bash ./scripts/cargo-install-all.sh . diff --git a/docs/src/clusters/benchmark.md b/docs/src/clusters/benchmark.md index 35978cdd09..424262caac 100644 --- a/docs/src/clusters/benchmark.md +++ b/docs/src/clusters/benchmark.md @@ -6,7 +6,7 @@ The Solana git repository contains all the scripts you might need to spin up you For all four variations, you'd need the latest Rust toolchain and the Solana source code: -First, setup Rust, Cargo and system packages as described in the Solana [README](https://github.com/solana-labs/solana#1-install-rustc-cargo-and-rustfmt) +First, setup Rust, Cargo and system packages as described in the Solana [README](https://github.com/jito-foundation/jito-solana#1-install-rustc-cargo-and-rustfmt) Now checkout the code from github: diff --git a/docs/src/implemented-proposals/installer.md b/docs/src/implemented-proposals/installer.md index c052aa7b4e..343fa67fc3 100644 --- a/docs/src/implemented-proposals/installer.md +++ b/docs/src/implemented-proposals/installer.md @@ -2,9 +2,12 @@ title: Cluster Software Installation and Updates --- -Currently users are required to build the solana cluster software themselves from the git repository and manually update it, which is error prone and inconvenient. +Currently users are required to build the solana cluster software themselves from the git repository and manually update +it, which is error prone and inconvenient. -This document proposes an easy to use software install and updater that can be used to deploy pre-built binaries for supported platforms. Users may elect to use binaries supplied by Solana or any other party provider. Deployment of updates is managed using an on-chain update manifest program. +This document proposes an easy to use software install and updater that can be used to deploy pre-built binaries for +supported platforms. Users may elect to use binaries supplied by Solana or any other party provider. Deployment of +updates is managed using an on-chain update manifest program. ## Motivating Examples @@ -13,16 +16,17 @@ This document proposes an easy to use software install and updater that can be u The easiest install method for supported platforms: ```bash -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh +$ curl -sSf https://raw.githubusercontent.com/jito-foundation/jito-solana/v1.0.0/install/agave-install-init.sh | sh ``` -This script will check github for the latest tagged release and download and run the `agave-install-init` binary from there. +This script will check github for the latest tagged release and download and run the `agave-install-init` binary from +there. If additional arguments need to be specified during the installation, the following shell syntax is used: ```bash $ init_args=.... # arguments for `agave-install-init ...` -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh -s - ${init_args} +$ curl -sSf https://raw.githubusercontent.com/jito-foundation/jito-solana/v1.0.0/install/agave-install-init.sh | sh -s - ${init_args} ``` ### Fetch and run a pre-built installer from a Github release @@ -30,7 +34,7 @@ $ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/ With a well-known release URL, a pre-built binary can be obtained for supported platforms: ```bash -$ curl -o agave-install-init https://github.com/solana-labs/solana/releases/download/v1.0.0/agave-install-init-x86_64-apple-darwin +$ curl -o agave-install-init https://github.com/jito-foundation/jito-solana/releases/download/v1.0.0/agave-install-init-x86_64-apple-darwin $ chmod +x ./agave-install-init $ ./agave-install-init --help ``` @@ -40,14 +44,15 @@ $ ./agave-install-init --help If a pre-built binary is not available for a given platform, building the installer from source is always an option: ```bash -$ git clone https://github.com/solana-labs/solana.git +$ git clone https://github.com/jito-foundation/jito-solana.git $ cd solana/install $ cargo run -- --help ``` ### Deploy a new update to a cluster -Given a solana release tarball \(as created by `ci/publish-tarball.sh`\) that has already been uploaded to a publicly accessible URL, the following commands will deploy the update: +Given a solana release tarball \(as created by `ci/publish-tarball.sh`\) that has already been uploaded to a publicly +accessible URL, the following commands will deploy the update: ```bash $ solana-keygen new -o update-manifest.json # <-- only generated once, the public key is shared with users @@ -65,7 +70,10 @@ $ agave-install run agave-validator ... # <-- runs a validator, restarting it a ## On-chain Update Manifest -An update manifest is used to advertise the deployment of new release tarballs on a solana cluster. The update manifest is stored using the `config` program, and each update manifest account describes a logical update channel for a given target triple \(eg, `x86_64-apple-darwin`\). The account public key is well-known between the entity deploying new updates and users consuming those updates. +An update manifest is used to advertise the deployment of new release tarballs on a solana cluster. The update manifest +is stored using the `config` program, and each update manifest account describes a logical update channel for a given +target triple \(eg, `x86_64-apple-darwin`\). The account public key is well-known between the entity deploying new +updates and users consuming those updates. The update tarball itself is hosted elsewhere, off-chain and can be fetched from the specified `download_url`. @@ -87,9 +95,11 @@ pub struct SignedUpdateManifest { } ``` -Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against man-in-the-middle attacks between the `agave-install` tool and the solana cluster RPC API. +Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against +man-in-the-middle attacks between the `agave-install` tool and the solana cluster RPC API. -To guard against rollback attacks, `agave-install` will refuse to install an update with an older `timestamp_secs` than what is currently installed. +To guard against rollback attacks, `agave-install` will refuse to install an update with an older `timestamp_secs` than +what is currently installed. ## Release Archive Contents @@ -116,7 +126,8 @@ The `agave-install` tool is used by the user to install and update their cluster It manages the following files and directories in the user's home directory: - `~/.config/solana/install/config.yml` - user configuration and information about currently installed software version -- `~/.local/share/solana/install/bin` - a symlink to the current release. eg, `~/.local/share/solana-update/-/bin` +- `~/.local/share/solana/install/bin` - a symlink to the current release. + eg, `~/.local/share/solana-update/-/bin` - `~/.local/share/solana/install/releases//` - contents of a release ### Command-line Interface diff --git a/entry/src/entry.rs b/entry/src/entry.rs index da4fda5914..ebf91dc644 100644 --- a/entry/src/entry.rs +++ b/entry/src/entry.rs @@ -220,7 +220,7 @@ pub fn hash_transactions(transactions: &[VersionedTransaction]) -> Hash { .iter() .flat_map(|tx| tx.signatures.iter()) .collect(); - let merkle_tree = MerkleTree::new(&signatures); + let merkle_tree = MerkleTree::new(&signatures, false); if let Some(root_hash) = merkle_tree.get_root() { *root_hash } else { diff --git a/entry/src/poh.rs b/entry/src/poh.rs index b54c8a745a..de14e835e1 100644 --- a/entry/src/poh.rs +++ b/entry/src/poh.rs @@ -72,19 +72,30 @@ impl Poh { } pub fn record(&mut self, mixin: Hash) -> Option { - if self.remaining_hashes == 1 { + let entries = self.record_bundle(&[mixin]); + entries.unwrap_or_default().pop() + } + + pub fn record_bundle(&mut self, mixins: &[Hash]) -> Option> { + if self.remaining_hashes <= mixins.len() as u64 { return None; // Caller needs to `tick()` first } - self.hash = hashv(&[self.hash.as_ref(), mixin.as_ref()]); - let num_hashes = self.num_hashes + 1; - self.num_hashes = 0; - self.remaining_hashes -= 1; + let entries = mixins + .iter() + .map(|m| { + self.hash = hashv(&[self.hash.as_ref(), m.as_ref()]); + let num_hashes = self.num_hashes + 1; + self.num_hashes = 0; + self.remaining_hashes -= 1; + PohEntry { + num_hashes, + hash: self.hash, + } + }) + .collect(); - Some(PohEntry { - num_hashes, - hash: self.hash, - }) + Some(entries) } pub fn tick(&mut self) -> Option { diff --git a/f b/f new file mode 100755 index 0000000000..e5fe635508 --- /dev/null +++ b/f @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Builds jito-solana in a docker container. +# Useful for running on machines that might not have cargo installed but can run docker (Flatcar Linux). +# run `./f true` to compile with debug flags + +set -eux + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +GIT_SHA="$(git rev-parse --short HEAD)" + +echo "Git hash: $GIT_SHA" + +DEBUG_FLAGS=${1-false} + +DOCKER_BUILDKIT=1 docker build \ + --build-arg debug=$DEBUG_FLAGS \ + --build-arg ci_commit=$GIT_SHA \ + -t jitolabs/build-solana \ + -f dev/Dockerfile . \ + --progress=plain + +# Creates a temporary container, copies solana-validator built inside container there and +# removes the temporary container. +docker rm temp || true +docker container create --name temp jitolabs/build-solana +mkdir -p $SCRIPT_DIR/docker-output +# Outputs the solana-validator binary to $SOLANA/docker-output/solana-validator +docker container cp temp:/solana/docker-output $SCRIPT_DIR/ +docker rm temp diff --git a/fetch-spl.sh b/fetch-spl.sh index 97fb1c50aa..97de447388 100755 --- a/fetch-spl.sh +++ b/fetch-spl.sh @@ -13,8 +13,24 @@ fetch_program() { declare version=$2 declare address=$3 declare loader=$4 + declare repo=$5 - declare so=spl_$name-$version.so + case $repo in + "jito") + so=$name-$version.so + so_name="$name.so" + url="https://github.com/jito-foundation/jito-programs/releases/download/v$version/$so_name" + ;; + "solana") + so=spl_$name-$version.so + so_name="spl_${name//-/_}.so" + url="https://github.com/solana-labs/solana-program-library/releases/download/$name-v$version/$so_name" + ;; + *) + echo "Unsupported repo: $repo" + return 1 + ;; + esac if [[ $loader == "$upgradeableLoader" ]]; then genesis_args+=(--upgradeable-program "$address" "$loader" "$so" none) @@ -30,12 +46,11 @@ fetch_program() { cp ~/.cache/solana-spl/"$so" "$so" else echo "Downloading $name $version" - so_name="spl_${name//-/_}.so" ( set -x curl -L --retry 5 --retry-delay 2 --retry-connrefused \ -o "$so" \ - "https://github.com/solana-labs/solana-program-library/releases/download/$name-v$version/$so_name" + "$url" ) mkdir -p ~/.cache/solana-spl @@ -44,19 +59,25 @@ fetch_program() { } -fetch_program token 3.5.0 TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA BPFLoader2111111111111111111111111111111111 -fetch_program token-2022 1.0.0 TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb BPFLoaderUpgradeab1e11111111111111111111111 -fetch_program memo 1.0.0 Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo BPFLoader1111111111111111111111111111111111 -fetch_program memo 3.0.0 MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr BPFLoader2111111111111111111111111111111111 -fetch_program associated-token-account 1.1.2 ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL BPFLoader2111111111111111111111111111111111 -fetch_program feature-proposal 1.0.0 Feat1YXHhH6t1juaWF74WLcfv4XoNocjXA6sPWHNgAse BPFLoader2111111111111111111111111111111111 +fetch_program token 3.5.0 TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA BPFLoader2111111111111111111111111111111111 solana +fetch_program token-2022 1.0.0 TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb BPFLoaderUpgradeab1e11111111111111111111111 solana +fetch_program memo 1.0.0 Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo BPFLoader1111111111111111111111111111111111 solana +fetch_program memo 3.0.0 MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr BPFLoader2111111111111111111111111111111111 solana +fetch_program associated-token-account 1.1.2 ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL BPFLoader2111111111111111111111111111111111 solana +fetch_program feature-proposal 1.0.0 Feat1YXHhH6t1juaWF74WLcfv4XoNocjXA6sPWHNgAse BPFLoader2111111111111111111111111111111111 solana +# jito programs +fetch_program jito_tip_payment 0.1.4 T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt BPFLoaderUpgradeab1e11111111111111111111111 jito +fetch_program jito_tip_distribution 0.1.4 4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7 BPFLoaderUpgradeab1e11111111111111111111111 jito -echo "${genesis_args[@]}" > spl-genesis-args.sh +echo "${genesis_args[@]}" >spl-genesis-args.sh echo echo "Available SPL programs:" ls -l spl_*.so +echo "Available Jito programs:" +ls -l jito*.so + echo echo "solana-genesis command-line arguments (spl-genesis-args.sh):" cat spl-genesis-args.sh diff --git a/gossip/src/cluster_info.rs b/gossip/src/cluster_info.rs index c1142096fb..c97757f16c 100644 --- a/gossip/src/cluster_info.rs +++ b/gossip/src/cluster_info.rs @@ -551,6 +551,10 @@ impl ClusterInfo { *self.entrypoints.write().unwrap() = entrypoints; } + pub fn set_my_contact_info(&self, my_contact_info: ContactInfo) { + *self.my_contact_info.write().unwrap() = my_contact_info; + } + pub fn save_contact_info(&self) { let nodes = { let entrypoint_gossip_addrs = self diff --git a/install/agave-install-init.sh b/install/agave-install-init.sh index cf2d1babf3..8fa5dbbcdf 100755 --- a/install/agave-install-init.sh +++ b/install/agave-install-init.sh @@ -16,9 +16,9 @@ { # this ensures the entire script is downloaded # if [ -z "$SOLANA_DOWNLOAD_ROOT" ]; then - SOLANA_DOWNLOAD_ROOT="https://github.com/anza-xyz/agave/releases/download/" + SOLANA_DOWNLOAD_ROOT="https://github.com/jito-foundation/jito-solana/releases/download/" fi -GH_LATEST_RELEASE="https://api.github.com/repos/anza-xyz/agave/releases/latest" +GH_LATEST_RELEASE="https://api.github.com/repos/jito-foundation/jito-solana/releases/latest" set -e diff --git a/install/src/command.rs b/install/src/command.rs index 8a81e1d723..09165562cf 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -568,7 +568,7 @@ pub fn init( fn github_release_download_url(release_semver: &str) -> String { format!( - "https://github.com/anza-xyz/agave/releases/download/v{}/solana-release-{}.tar.bz2", + "https://github.com/jito-foundation/jito-solana/releases/download/v{}/solana-release-{}.tar.bz2", release_semver, crate::build_env::TARGET ) @@ -576,7 +576,7 @@ fn github_release_download_url(release_semver: &str) -> String { fn release_channel_download_url(release_channel: &str) -> String { format!( - "https://release.anza.xyz/{}/solana-release-{}.tar.bz2", + "https://release.jito.wtf/{}/solana-release-{}.tar.bz2", release_channel, crate::build_env::TARGET ) @@ -584,7 +584,7 @@ fn release_channel_download_url(release_channel: &str) -> String { fn release_channel_version_url(release_channel: &str) -> String { format!( - "https://release.anza.xyz/{}/solana-release-{}.yml", + "https://release.jito.wtf/{}/solana-release-{}.yml", release_channel, crate::build_env::TARGET ) @@ -901,7 +901,7 @@ fn check_for_newer_github_release( while page == 1 || releases.len() == PER_PAGE { let url = reqwest::Url::parse_with_params( - "https://api.github.com/repos/anza-xyz/agave/releases", + "https://api.github.com/repos/jito-foundation/jito-solana/releases", &[ ("per_page", &format!("{PER_PAGE}")), ("page", &format!("{page}")), diff --git a/jito-programs b/jito-programs new file mode 160000 index 0000000000..d2b9c58189 --- /dev/null +++ b/jito-programs @@ -0,0 +1 @@ +Subproject commit d2b9c58189bb69d6f90b1ed513beea8cc9d7c013 diff --git a/jito-protos/Cargo.toml b/jito-protos/Cargo.toml new file mode 100644 index 0000000000..f9f0b5baa3 --- /dev/null +++ b/jito-protos/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "jito-protos" +version = { workspace = true } +edition = { workspace = true } +publish = false + +[dependencies] +bytes = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } +tonic = { workspace = true } + +[build-dependencies] +tonic-build = { workspace = true } + +# windows users should install the protobuf compiler manually and set the PROTOC +# envar to point to the installed binary +[target."cfg(not(windows))".build-dependencies] +protobuf-src = { workspace = true } diff --git a/jito-protos/build.rs b/jito-protos/build.rs new file mode 100644 index 0000000000..30ece1620a --- /dev/null +++ b/jito-protos/build.rs @@ -0,0 +1,38 @@ +use tonic_build::configure; + +fn main() -> Result<(), std::io::Error> { + const PROTOC_ENVAR: &str = "PROTOC"; + if std::env::var(PROTOC_ENVAR).is_err() { + #[cfg(not(windows))] + std::env::set_var(PROTOC_ENVAR, protobuf_src::protoc()); + } + + let proto_base_path = std::path::PathBuf::from("protos"); + let proto_files = [ + "auth.proto", + "block_engine.proto", + "bundle.proto", + "packet.proto", + "relayer.proto", + "shared.proto", + ]; + let mut protos = Vec::new(); + for proto_file in &proto_files { + let proto = proto_base_path.join(proto_file); + println!("cargo:rerun-if-changed={}", proto.display()); + protos.push(proto); + } + + configure() + .build_client(true) + .build_server(false) + .type_attribute( + "TransactionErrorType", + "#[cfg_attr(test, derive(enum_iterator::Sequence))]", + ) + .type_attribute( + "InstructionErrorType", + "#[cfg_attr(test, derive(enum_iterator::Sequence))]", + ) + .compile(&protos, &[proto_base_path]) +} diff --git a/jito-protos/protos b/jito-protos/protos new file mode 160000 index 0000000000..05d210980f --- /dev/null +++ b/jito-protos/protos @@ -0,0 +1 @@ +Subproject commit 05d210980f34a7c974d7ed1a4dbcb2ce1fca00b3 diff --git a/jito-protos/src/lib.rs b/jito-protos/src/lib.rs new file mode 100644 index 0000000000..cf630c53d2 --- /dev/null +++ b/jito-protos/src/lib.rs @@ -0,0 +1,25 @@ +pub mod proto { + pub mod auth { + tonic::include_proto!("auth"); + } + + pub mod block_engine { + tonic::include_proto!("block_engine"); + } + + pub mod bundle { + tonic::include_proto!("bundle"); + } + + pub mod packet { + tonic::include_proto!("packet"); + } + + pub mod relayer { + tonic::include_proto!("relayer"); + } + + pub mod shared { + tonic::include_proto!("shared"); + } +} diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index a79645e428..e195ac1cb6 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -1380,6 +1380,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) { blockstore.clone(), process_options, None, + true, ); let bank = bank_forks.read().unwrap().working_bank(); diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 98a647e21f..32d0f6ce05 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -112,6 +112,7 @@ pub fn load_and_process_ledger_or_exit( blockstore: Arc, process_options: ProcessOptions, transaction_status_sender: Option, + ignore_halt_at_slot_for_snapshot_loading: bool, ) -> LoadAndProcessLedgerOutput { load_and_process_ledger( arg_matches, @@ -119,6 +120,7 @@ pub fn load_and_process_ledger_or_exit( blockstore, process_options, transaction_status_sender, + ignore_halt_at_slot_for_snapshot_loading, ) .unwrap_or_else(|err| { eprintln!("Exiting. Failed to load and process ledger: {err}"); @@ -132,6 +134,7 @@ pub fn load_and_process_ledger( blockstore: Arc, process_options: ProcessOptions, transaction_status_sender: Option, + ignore_halt_at_slot_for_snapshot_loading: bool, ) -> Result { let bank_snapshots_dir = if blockstore.is_primary_access() { blockstore.ledger_path().join("snapshot") @@ -142,6 +145,12 @@ pub fn load_and_process_ledger( .join("snapshot") }; + let snapshot_halt_at_slot = if ignore_halt_at_slot_for_snapshot_loading { + None + } else { + process_options.halt_at_slot + }; + let mut starting_slot = 0; // default start check with genesis let snapshot_config = if arg_matches.is_present("no_snapshot") { None @@ -155,13 +164,15 @@ pub fn load_and_process_ledger( .ok() .map(PathBuf::from) .unwrap_or_else(|| full_snapshot_archives_dir.clone()); - if let Some(full_snapshot_slot) = - snapshot_utils::get_highest_full_snapshot_archive_slot(&full_snapshot_archives_dir) - { + if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot( + &full_snapshot_archives_dir, + snapshot_halt_at_slot, + ) { let incremental_snapshot_slot = snapshot_utils::get_highest_incremental_snapshot_archive_slot( &incremental_snapshot_archives_dir, full_snapshot_slot, + snapshot_halt_at_slot, ) .unwrap_or_default(); starting_slot = std::cmp::max(full_snapshot_slot, incremental_snapshot_slot); @@ -294,6 +305,7 @@ pub fn load_and_process_ledger( None, // Maybe support this later, though accounts_update_notifier, exit.clone(), + ignore_halt_at_slot_for_snapshot_loading, ) .map_err(LoadAndProcessLedgerError::LoadBankForks)?; let block_verification_method = value_t!( diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 8339c0a14d..ca5f16f0cb 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1436,8 +1436,8 @@ fn main() { Arc::new(blockstore), process_options, None, + true, ); - println!( "{}", compute_shred_version( @@ -1630,6 +1630,7 @@ fn main() { Arc::new(blockstore), process_options, transaction_status_sender, + true, ); let working_bank = bank_forks.read().unwrap().working_bank(); @@ -1697,6 +1698,7 @@ fn main() { Arc::new(blockstore), process_options, None, + true, ); let dot = graph_forks(&bank_forks.read().unwrap(), &graph_config); @@ -1870,6 +1872,7 @@ fn main() { blockstore.clone(), process_options, None, + false, ); // Snapshot creation will implicitly perform AccountsDb // flush and clean operations. These operations cannot be @@ -2265,6 +2268,7 @@ fn main() { Arc::new(blockstore), process_options, None, + true, ); let bank = bank_forks.read().unwrap().working_bank(); @@ -2317,7 +2321,9 @@ fn main() { Arc::new(blockstore), process_options, None, + true, ); + let bank_forks = bank_forks.read().unwrap(); let slot = bank_forks.working_bank().slot(); let bank = bank_forks.get(slot).unwrap_or_else(|| { diff --git a/ledger-tool/src/program.rs b/ledger-tool/src/program.rs index 463d017b17..e240f9d845 100644 --- a/ledger-tool/src/program.rs +++ b/ledger-tool/src/program.rs @@ -85,6 +85,7 @@ fn load_blockstore(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> Arc, accounts_update_notifier: Option, exit: Arc, + ignore_halt_at_slot_for_snapshot_loading: bool, ) -> LoadResult { fn get_snapshots_to_load( snapshot_config: Option<&SnapshotConfig>, + halt_at_slot: Option, + ignore_halt_at_slot_for_snapshot_loading: bool, ) -> Option<( FullSnapshotArchiveInfo, Option, @@ -137,9 +141,16 @@ pub fn load_bank_forks( return None; }; + let halt_at_slot = if ignore_halt_at_slot_for_snapshot_loading { + None + } else { + halt_at_slot + }; + let Some(full_snapshot_archive_info) = snapshot_utils::get_highest_full_snapshot_archive_info( &snapshot_config.full_snapshot_archives_dir, + halt_at_slot, ) else { warn!( @@ -153,6 +164,7 @@ pub fn load_bank_forks( snapshot_utils::get_highest_incremental_snapshot_archive_info( &snapshot_config.incremental_snapshot_archives_dir, full_snapshot_archive_info.slot(), + halt_at_slot, ); Some(( @@ -163,7 +175,11 @@ pub fn load_bank_forks( let (bank_forks, starting_snapshot_hashes) = if let Some((full_snapshot_archive_info, incremental_snapshot_archive_info)) = - get_snapshots_to_load(snapshot_config) + get_snapshots_to_load( + snapshot_config, + process_options.halt_at_slot, + ignore_halt_at_slot_for_snapshot_loading, + ) { // SAFETY: Having snapshots to load ensures a snapshot config let snapshot_config = snapshot_config.unwrap(); @@ -222,7 +238,7 @@ pub fn load_bank_forks( } #[allow(clippy::too_many_arguments)] -fn bank_forks_from_snapshot( +pub fn bank_forks_from_snapshot( full_snapshot_archive_info: FullSnapshotArchiveInfo, incremental_snapshot_archive_info: Option, genesis_config: &GenesisConfig, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index f5be5afd0b..1a22d7c9c8 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -159,7 +159,7 @@ pub fn execute_batch( let mut mint_decimals: HashMap = HashMap::new(); let pre_token_balances = if record_token_balances { - collect_token_balances(bank, batch, &mut mint_decimals) + collect_token_balances(bank, batch, &mut mint_decimals, None) } else { vec![] }; @@ -217,7 +217,7 @@ pub fn execute_batch( if let Some(transaction_status_sender) = transaction_status_sender { let transactions = batch.sanitized_transactions().to_vec(); let post_token_balances = if record_token_balances { - collect_token_balances(bank, batch, &mut mint_decimals) + collect_token_balances(bank, batch, &mut mint_decimals, None) } else { vec![] }; @@ -851,6 +851,7 @@ pub fn test_process_blockstore( None, None, exit, + true, ) .unwrap(); diff --git a/ledger/src/token_balances.rs b/ledger/src/token_balances.rs index cc074dfcc0..0055bffa1e 100644 --- a/ledger/src/token_balances.rs +++ b/ledger/src/token_balances.rs @@ -7,6 +7,7 @@ use { solana_metrics::datapoint_debug, solana_runtime::{bank::Bank, transaction_batch::TransactionBatch}, solana_sdk::{account::ReadableAccount, pubkey::Pubkey}, + solana_svm::account_overrides::AccountOverrides, solana_transaction_status::{ token_balances::TransactionTokenBalances, TransactionTokenBalance, }, @@ -39,6 +40,7 @@ pub fn collect_token_balances( bank: &Bank, batch: &TransactionBatch, mint_decimals: &mut HashMap, + cached_accounts: Option<&AccountOverrides>, ) -> TransactionTokenBalances { let mut balances: TransactionTokenBalances = vec![]; let mut collect_time = Measure::start("collect_token_balances"); @@ -59,8 +61,12 @@ pub fn collect_token_balances( ui_token_amount, owner, program_id, - }) = collect_token_balance_from_account(bank, account_id, mint_decimals) - { + }) = collect_token_balance_from_account( + bank, + account_id, + mint_decimals, + cached_accounts, + ) { transaction_balances.push(TransactionTokenBalance { account_index: index as u8, mint, @@ -93,8 +99,17 @@ fn collect_token_balance_from_account( bank: &Bank, account_id: &Pubkey, mint_decimals: &mut HashMap, + account_overrides: Option<&AccountOverrides>, ) -> Option { - let account = bank.get_account(account_id)?; + let account = { + if let Some(account_override) = + account_overrides.and_then(|overrides| overrides.get(account_id)) + { + Some(account_override.clone()) + } else { + bank.get_account(account_id) + } + }?; if !is_known_spl_token_id(account.owner()) { return None; @@ -243,13 +258,13 @@ mod test { // Account is not owned by spl_token (nor does it have TokenAccount state) assert_eq!( - collect_token_balance_from_account(&bank, &account_pubkey, &mut mint_decimals), + collect_token_balance_from_account(&bank, &account_pubkey, &mut mint_decimals, None), None ); // Mint does not have TokenAccount state assert_eq!( - collect_token_balance_from_account(&bank, &mint_pubkey, &mut mint_decimals), + collect_token_balance_from_account(&bank, &mint_pubkey, &mut mint_decimals, None), None ); @@ -258,7 +273,8 @@ mod test { collect_token_balance_from_account( &bank, &spl_token_account_pubkey, - &mut mint_decimals + &mut mint_decimals, + None ), Some(TokenBalanceData { mint: mint_pubkey.to_string(), @@ -275,7 +291,12 @@ mod test { // TokenAccount is not owned by known spl-token program_id assert_eq!( - collect_token_balance_from_account(&bank, &other_account_pubkey, &mut mint_decimals), + collect_token_balance_from_account( + &bank, + &other_account_pubkey, + &mut mint_decimals, + None + ), None ); @@ -284,7 +305,8 @@ mod test { collect_token_balance_from_account( &bank, &other_mint_account_pubkey, - &mut mint_decimals + &mut mint_decimals, + None ), None ); @@ -437,13 +459,13 @@ mod test { // Account is not owned by spl_token (nor does it have TokenAccount state) assert_eq!( - collect_token_balance_from_account(&bank, &account_pubkey, &mut mint_decimals), + collect_token_balance_from_account(&bank, &account_pubkey, &mut mint_decimals, None), None ); // Mint does not have TokenAccount state assert_eq!( - collect_token_balance_from_account(&bank, &mint_pubkey, &mut mint_decimals), + collect_token_balance_from_account(&bank, &mint_pubkey, &mut mint_decimals, None), None ); @@ -452,7 +474,8 @@ mod test { collect_token_balance_from_account( &bank, &spl_token_account_pubkey, - &mut mint_decimals + &mut mint_decimals, + None ), Some(TokenBalanceData { mint: mint_pubkey.to_string(), @@ -469,7 +492,12 @@ mod test { // TokenAccount is not owned by known spl-token program_id assert_eq!( - collect_token_balance_from_account(&bank, &other_account_pubkey, &mut mint_decimals), + collect_token_balance_from_account( + &bank, + &other_account_pubkey, + &mut mint_decimals, + None + ), None ); @@ -478,7 +506,8 @@ mod test { collect_token_balance_from_account( &bank, &other_mint_account_pubkey, - &mut mint_decimals + &mut mint_decimals, + None ), None ); diff --git a/local-cluster/src/local_cluster.rs b/local-cluster/src/local_cluster.rs index 9374e93770..45d793f23f 100644 --- a/local-cluster/src/local_cluster.rs +++ b/local-cluster/src/local_cluster.rs @@ -342,6 +342,7 @@ impl LocalCluster { true, 32, // max connections per IpAddr per minute Arc::new(RwLock::new(None)), + None, ) .expect("assume successful validator start"); @@ -544,6 +545,7 @@ impl LocalCluster { DEFAULT_TPU_ENABLE_UDP, 32, // max connections per IpAddr per mintute Arc::new(RwLock::new(None)), + None, ) .expect("assume successful validator start"); @@ -1090,6 +1092,7 @@ impl Cluster for LocalCluster { DEFAULT_TPU_ENABLE_UDP, 32, // max connections per IpAddr per minute, use higher value because of tests Arc::new(RwLock::new(None)), + None, ) .expect("assume successful validator start"); cluster_validator_info.validator = Some(restarted_node); diff --git a/local-cluster/src/local_cluster_snapshot_utils.rs b/local-cluster/src/local_cluster_snapshot_utils.rs index 3df5b61d3b..37aaf9ec12 100644 --- a/local-cluster/src/local_cluster_snapshot_utils.rs +++ b/local-cluster/src/local_cluster_snapshot_utils.rs @@ -91,7 +91,10 @@ impl LocalCluster { let timer = Instant::now(); let next_snapshot = loop { if let Some(full_snapshot_archive_info) = - snapshot_utils::get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir) + snapshot_utils::get_highest_full_snapshot_archive_info( + &full_snapshot_archives_dir, + None, + ) { match next_snapshot_type { NextSnapshotType::FullSnapshot => { @@ -104,6 +107,7 @@ impl LocalCluster { snapshot_utils::get_highest_incremental_snapshot_archive_info( incremental_snapshot_archives_dir.as_ref().unwrap(), full_snapshot_archive_info.slot(), + None, ) { if incremental_snapshot_archive_info.slot() >= last_slot { diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index a2366eb414..05971e1f02 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -73,6 +73,11 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { replay_forks_threads: config.replay_forks_threads, replay_transactions_threads: config.replay_transactions_threads, delay_leader_block_for_pending_fork: config.delay_leader_block_for_pending_fork, + relayer_config: config.relayer_config.clone(), + block_engine_config: config.block_engine_config.clone(), + shred_receiver_address: config.shred_receiver_address.clone(), + tip_manager_config: config.tip_manager_config.clone(), + preallocated_bundle_cost: config.preallocated_bundle_cost, } } diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 62f7fd3243..4ec5e56896 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -844,6 +844,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st validator_snapshot_test_config .full_snapshot_archives_dir .path(), + None, ) .unwrap(); info!( @@ -883,6 +884,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st .incremental_snapshot_archives_dir .path(), full_snapshot_archive.slot(), + None, ) .unwrap(); info!( @@ -1026,6 +1028,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st validator_snapshot_test_config .full_snapshot_archives_dir .path(), + None, ) .unwrap(); @@ -1092,6 +1095,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st validator_snapshot_test_config .full_snapshot_archives_dir .path(), + None, ) .unwrap(); @@ -1120,6 +1124,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st validator_snapshot_test_config .full_snapshot_archives_dir .path(), + None, ) { if full_snapshot_slot >= validator_next_full_snapshot_slot { if let Some(incremental_snapshot_slot) = @@ -1128,6 +1133,7 @@ fn test_incremental_snapshot_download_with_crossing_full_snapshot_interval_at_st .incremental_snapshot_archives_dir .path(), full_snapshot_slot, + None, ) { if incremental_snapshot_slot >= validator_next_incremental_snapshot_slot { @@ -1322,8 +1328,10 @@ fn test_snapshots_blockstore_floor() { trace!("Waiting for snapshot tar to be generated with slot",); let archive_info = loop { - let archive = - snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir); + let archive = snapshot_utils::get_highest_full_snapshot_archive_info( + full_snapshot_archives_dir, + None, + ); if archive.is_some() { trace!("snapshot exists"); break archive.unwrap(); @@ -5079,12 +5087,14 @@ fn test_boot_from_local_state() { if let Some(other_full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot( &other_validator_config.full_snapshot_archives_dir, + None, ) { let other_incremental_snapshot_slot = snapshot_utils::get_highest_incremental_snapshot_archive_slot( &other_validator_config.incremental_snapshot_archives_dir, other_full_snapshot_slot, + None, ); if other_full_snapshot_slot >= full_snapshot_archive.slot() && other_incremental_snapshot_slot >= Some(incremental_snapshot_archive.slot()) @@ -5234,6 +5244,7 @@ fn test_boot_from_local_state_missing_archive() { info!("Deleting latest full snapshot archive..."); let highest_full_snapshot = snapshot_utils::get_highest_full_snapshot_archive_info( validator_config.full_snapshot_archives_dir.path(), + None, ) .unwrap(); fs::remove_file(highest_full_snapshot.path()).unwrap(); diff --git a/merkle-tree/src/merkle_tree.rs b/merkle-tree/src/merkle_tree.rs index 09285a41e7..57bceea1ec 100644 --- a/merkle-tree/src/merkle_tree.rs +++ b/merkle-tree/src/merkle_tree.rs @@ -18,7 +18,7 @@ macro_rules! hash_intermediate { } } -#[derive(Debug)] +#[derive(Default, Debug, Eq, Hash, PartialEq)] pub struct MerkleTree { leaf_count: usize, nodes: Vec, @@ -36,6 +36,14 @@ impl<'a> ProofEntry<'a> { assert!(left_sibling.is_none() ^ right_sibling.is_none()); Self(target, left_sibling, right_sibling) } + + pub fn get_left_sibling(&self) -> Option<&'a Hash> { + self.1 + } + + pub fn get_right_sibling(&self) -> Option<&'a Hash> { + self.2 + } } #[derive(Debug, Default, PartialEq, Eq)] @@ -60,6 +68,10 @@ impl<'a> Proof<'a> { }); result.is_some() } + + pub fn get_proof_entries(self) -> Vec> { + self.0 + } } impl MerkleTree { @@ -95,7 +107,7 @@ impl MerkleTree { } } - pub fn new>(items: &[T]) -> Self { + pub fn new>(items: &[T], sorted_hashes: bool) -> Self { let cap = MerkleTree::calculate_vec_capacity(items.len()); let mut mt = MerkleTree { leaf_count: items.len(), @@ -123,8 +135,20 @@ impl MerkleTree { &mt.nodes[prev_level_start + prev_level_idx] }; - let hash = hash_intermediate!(lsib, rsib); - mt.nodes.push(hash); + // tip-distribution verification uses sorted hashing + if sorted_hashes { + if lsib <= rsib { + let hash = hash_intermediate!(lsib, rsib); + mt.nodes.push(hash); + } else { + let hash = hash_intermediate!(rsib, lsib); + mt.nodes.push(hash); + } + } else { + // hashing for solana internals + let hash = hash_intermediate!(lsib, rsib); + mt.nodes.push(hash); + } } prev_level_start = level_start; prev_level_len = level_len; @@ -189,21 +213,21 @@ mod tests { #[test] fn test_tree_from_empty() { - let mt = MerkleTree::new::<[u8; 0]>(&[]); + let mt = MerkleTree::new::<[u8; 0]>(&[], false); assert_eq!(mt.get_root(), None); } #[test] fn test_tree_from_one() { let input = b"test"; - let mt = MerkleTree::new(&[input]); + let mt = MerkleTree::new(&[input], false); let expected = hash_leaf!(input); assert_eq!(mt.get_root(), Some(&expected)); } #[test] fn test_tree_from_many() { - let mt = MerkleTree::new(TEST); + let mt = MerkleTree::new(TEST, false); // This golden hash will need to be updated whenever the contents of `TEST` change in any // way, including addition, removal and reordering or any of the tree calculation algo // changes @@ -215,7 +239,7 @@ mod tests { #[test] fn test_path_creation() { - let mt = MerkleTree::new(TEST); + let mt = MerkleTree::new(TEST, false); for (i, _s) in TEST.iter().enumerate() { let _path = mt.find_path(i).unwrap(); } @@ -223,13 +247,13 @@ mod tests { #[test] fn test_path_creation_bad_index() { - let mt = MerkleTree::new(TEST); + let mt = MerkleTree::new(TEST, false); assert_eq!(mt.find_path(TEST.len()), None); } #[test] fn test_path_verify_good() { - let mt = MerkleTree::new(TEST); + let mt = MerkleTree::new(TEST, false); for (i, s) in TEST.iter().enumerate() { let hash = hash_leaf!(s); let path = mt.find_path(i).unwrap(); @@ -239,7 +263,7 @@ mod tests { #[test] fn test_path_verify_bad() { - let mt = MerkleTree::new(TEST); + let mt = MerkleTree::new(TEST, false); for (i, s) in BAD.iter().enumerate() { let hash = hash_leaf!(s); let path = mt.find_path(i).unwrap(); diff --git a/multinode-demo/bootstrap-validator.sh b/multinode-demo/bootstrap-validator.sh index 374a9288f1..65b317eca4 100755 --- a/multinode-demo/bootstrap-validator.sh +++ b/multinode-demo/bootstrap-validator.sh @@ -106,12 +106,42 @@ while [[ -n $1 ]]; do elif [[ $1 == --skip-require-tower ]]; then maybeRequireTower=false shift + elif [[ $1 == --relayer-url ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --block-engine-url ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --tip-payment-program-pubkey ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --tip-distribution-program-pubkey ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --commission-bps ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --shred-receiver-address ]]; then + args+=("$1" "$2") + shift 2 elif [[ $1 = --log-messages-bytes-limit ]]; then args+=("$1" "$2") shift 2 elif [[ $1 == --block-production-method ]]; then args+=("$1" "$2") shift 2 + elif [[ $1 == --geyser-plugin-config ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --trust-relayer-packets ]]; then + args+=("$1") + shift + elif [[ $1 == --rpc-threads ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --trust-block-engine-packets ]]; then + args+=("$1") + shift else echo "Unknown argument: $1" $program --help @@ -147,6 +177,7 @@ args+=( --no-incremental-snapshots --identity "$identity" --vote-account "$vote_account" + --merkle-root-upload-authority "$identity" --rpc-faucet-address 127.0.0.1:9900 --no-poh-speed-test --no-os-network-limits-test @@ -156,6 +187,9 @@ args+=( ) default_arg --gossip-port 8001 default_arg --log - +default_arg --tip-payment-program-pubkey "DThZmRNNXh7kvTQW9hXeGoWGPKktK8pgVAyoTLjH7UrT" +default_arg --tip-distribution-program-pubkey "FjrdANjvo76aCYQ4kf9FM1R8aESUcEE6F8V7qyoVUQcM" +default_arg --commission-bps 0 pid= diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index 77082f6589..a0271b8c95 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -85,6 +85,24 @@ while [[ -n $1 ]]; do vote_account=$2 args+=("$1" "$2") shift 2 + elif [[ $1 == --block-engine-url ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --relayer-url ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 = --merkle-root-upload-authority ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --tip-payment-program-pubkey ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --tip-distribution-program-pubkey ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --commission-bps ]]; then + args+=("$1" "$2") + shift 2 elif [[ $1 = --init-complete-file ]]; then args+=("$1" "$2") shift 2 @@ -182,6 +200,24 @@ while [[ -n $1 ]]; do elif [[ $1 == --block-production-method ]]; then args+=("$1" "$2") shift 2 + elif [[ $1 == --rpc-pubsub-enable-block-subscription ]]; then + args+=("$1") + shift + elif [[ $1 == --geyser-plugin-config ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --trust-relayer-packets ]]; then + args+=("$1") + shift + elif [[ $1 == --rpc-threads ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --shred-receiver-address ]]; then + args+=("$1" "$2") + shift 2 + elif [[ $1 == --trust-block-engine-packets ]]; then + args+=("$1") + shift elif [[ $1 = -h ]]; then usage "$@" else @@ -256,6 +292,10 @@ fi default_arg --identity "$identity" default_arg --vote-account "$vote_account" +default_arg --merkle-root-upload-authority "$identity" +default_arg --tip-payment-program-pubkey "DThZmRNNXh7kvTQW9hXeGoWGPKktK8pgVAyoTLjH7UrT" +default_arg --tip-distribution-program-pubkey "FjrdANjvo76aCYQ4kf9FM1R8aESUcEE6F8V7qyoVUQcM" +default_arg --commission-bps 0 default_arg --ledger "$ledger_dir" default_arg --log - default_arg --full-rpc-api diff --git a/perf/src/sigverify.rs b/perf/src/sigverify.rs index 0d29bfe571..59a8cbdcba 100644 --- a/perf/src/sigverify.rs +++ b/perf/src/sigverify.rs @@ -110,7 +110,7 @@ pub fn init() { /// Returns true if the signatrue on the packet verifies. /// Caller must do packet.set_discard(true) if this returns false. #[must_use] -fn verify_packet(packet: &mut Packet, reject_non_vote: bool) -> bool { +pub fn verify_packet(packet: &mut Packet, reject_non_vote: bool) -> bool { // If this packet was already marked as discard, drop it if packet.meta().discard() { return false; diff --git a/poh/src/poh_recorder.rs b/poh/src/poh_recorder.rs index 3ac35335c7..e909ab0f62 100644 --- a/poh/src/poh_recorder.rs +++ b/poh/src/poh_recorder.rs @@ -58,9 +58,14 @@ pub enum PohRecorderError { SendError(#[from] SendError), } -type Result = std::result::Result; +pub type Result = std::result::Result; -pub type WorkingBankEntry = (Arc, (Entry, u64)); +#[derive(Clone, Debug)] +pub struct WorkingBankEntry { + pub bank: Arc, + // normal entries have len == 1, bundles have len > 1 + pub entries_ticks: Vec<(Entry, u64)>, +} #[derive(Debug, Clone)] pub struct BankStart { @@ -90,21 +95,19 @@ impl BankStart { type RecordResultSender = Sender>>; pub struct Record { - pub mixin: Hash, - pub transactions: Vec, + // non-bundles shall have mixins_txs.len() == 1, bundles shall have mixins_txs.len() > 1 + pub mixins_txs: Vec<(Hash, Vec)>, pub slot: Slot, pub sender: RecordResultSender, } impl Record { pub fn new( - mixin: Hash, - transactions: Vec, + mixins_txs: Vec<(Hash, Vec)>, slot: Slot, sender: RecordResultSender, ) -> Self { Self { - mixin, - transactions, + mixins_txs, slot, sender, } @@ -158,16 +161,21 @@ impl TransactionRecorder { pub fn record_transactions( &self, bank_slot: Slot, - transactions: Vec, + batches: Vec>, ) -> RecordTransactionsSummary { let mut record_transactions_timings = RecordTransactionsTimings::default(); let mut starting_transaction_index = None; - if !transactions.is_empty() { - let (hash, hash_us) = measure_us!(hash_transactions(&transactions)); + if !batches.is_empty() && !batches.iter().any(|b| b.is_empty()) { + let (hashes, hash_us) = measure_us!(batches + .iter() + .map(|b| hash_transactions(b)) + .collect::>()); record_transactions_timings.hash_us = hash_us; - let (res, poh_record_us) = measure_us!(self.record(bank_slot, hash, transactions)); + let hashes_transactions: Vec<_> = hashes.into_iter().zip(batches).collect(); + + let (res, poh_record_us) = measure_us!(self.record(bank_slot, hashes_transactions)); record_transactions_timings.poh_record_us = poh_record_us; match res { @@ -203,14 +211,13 @@ impl TransactionRecorder { pub fn record( &self, bank_slot: Slot, - mixin: Hash, - transactions: Vec, + mixins_txs: Vec<(Hash, Vec)>, ) -> Result> { // create a new channel so that there is only 1 sender and when it goes out of scope, the receiver fails let (result_sender, result_receiver) = unbounded(); - let res = - self.record_sender - .send(Record::new(mixin, transactions, bank_slot, result_sender)); + let res = self + .record_sender + .send(Record::new(mixins_txs, bank_slot, result_sender)); if res.is_err() { // If the channel is dropped, then the validator is shutting down so return that we are hitting // the max tick height to stop transaction processing and flush any transactions in the pipeline. @@ -762,7 +769,10 @@ impl PohRecorder { for tick in &self.tick_cache[..entry_count] { working_bank.bank.register_tick(&tick.0.hash); - send_result = self.sender.send((working_bank.bank.clone(), tick.clone())); + send_result = self.sender.send(WorkingBankEntry { + bank: working_bank.bank.clone(), + entries_ticks: vec![tick.clone()], + }); if send_result.is_err() { break; } @@ -942,16 +952,23 @@ impl PohRecorder { pub fn record( &mut self, bank_slot: Slot, - mixin: Hash, - transactions: Vec, + mixins_txs: &[(Hash, Vec)], ) -> Result> { // Entries without transactions are used to track real-time passing in the ledger and // cannot be generated by `record()` - assert!(!transactions.is_empty(), "No transactions provided"); + assert!(!mixins_txs.is_empty(), "No transactions provided"); + assert!( + !mixins_txs.iter().any(|(_, txs)| txs.is_empty()), + "One of mixins is missing txs" + ); let ((), report_metrics_time) = measure!(self.report_metrics(bank_slot), "report_metrics"); self.report_metrics_us += report_metrics_time.as_us(); + let mixins: Vec = mixins_txs.iter().map(|(m, _)| *m).collect(); + let transactions: Vec> = + mixins_txs.iter().map(|(_, tx)| tx.clone()).collect(); + loop { let (flush_cache_res, flush_cache_time) = measure!(self.flush_cache(false), "flush_cache"); @@ -969,23 +986,36 @@ impl PohRecorder { let (mut poh_lock, poh_lock_time) = measure!(self.poh.lock().unwrap(), "poh_lock"); self.record_lock_contention_us += poh_lock_time.as_us(); - let (record_mixin_res, record_mixin_time) = - measure!(poh_lock.record(mixin), "record_mixin"); + let (maybe_entries, record_mixin_time) = + measure!(poh_lock.record_bundle(&mixins), "record_mixin"); self.record_us += record_mixin_time.as_us(); drop(poh_lock); - if let Some(poh_entry) = record_mixin_res { - let num_transactions = transactions.len(); + if let Some(entries) = maybe_entries { + assert_eq!(entries.len(), transactions.len()); + let num_transactions = transactions.iter().map(|txs| txs.len()).sum(); let (send_entry_res, send_entry_time) = measure!( { - let entry = Entry { - num_hashes: poh_entry.num_hashes, - hash: poh_entry.hash, - transactions, - }; + let entries_tick_heights: Vec<(Entry, u64)> = entries + .into_iter() + .zip(transactions.into_iter()) + .map(|(poh_entry, transactions)| { + ( + Entry { + num_hashes: poh_entry.num_hashes, + hash: poh_entry.hash, + transactions, + }, + self.tick_height, + ) + }) + .collect(); let bank_clone = working_bank.bank.clone(); - self.sender.send((bank_clone, (entry, self.tick_height))) + self.sender.send(WorkingBankEntry { + bank: bank_clone, + entries_ticks: entries_tick_heights, + }) }, "send_poh_entry", ); @@ -1349,7 +1379,11 @@ mod tests { assert_eq!(poh_recorder.tick_height, tick_height_before + 1); assert_eq!(poh_recorder.tick_cache.len(), 0); let mut num_entries = 0; - while let Ok((wbank, (_entry, _tick_height))) = entry_receiver.try_recv() { + while let Ok(WorkingBankEntry { + bank: wbank, + entries_ticks: _, + }) = entry_receiver.try_recv() + { assert_eq!(wbank.slot(), bank1.slot()); num_entries += 1; } @@ -1436,7 +1470,7 @@ mod tests { // We haven't yet reached the minimum tick height for the working bank, // so record should fail assert_matches!( - poh_recorder.record(bank1.slot(), h1, vec![tx.into()]), + poh_recorder.record(bank1.slot(), &[(h1, vec![tx.into()])]), Err(PohRecorderError::MinHeightNotReached) ); assert!(entry_receiver.try_recv().is_err()); @@ -1475,7 +1509,7 @@ mod tests { // However we hand over a bad slot so record fails let bad_slot = bank.slot() + 1; assert_matches!( - poh_recorder.record(bad_slot, h1, vec![tx.into()]), + poh_recorder.record(bad_slot, &[(h1, vec![tx.into()])]), Err(PohRecorderError::MaxHeightReached) ); } @@ -1518,18 +1552,26 @@ mod tests { let tx = test_tx(); let h1 = hash(b"hello world!"); assert!(poh_recorder - .record(bank1.slot(), h1, vec![tx.into()]) + .record(bank1.slot(), &[(h1, vec![tx.into()])]) .is_ok()); assert_eq!(poh_recorder.tick_cache.len(), 0); //tick in the cache + entry for _ in 0..min_tick_height { - let (_bank, (e, _tick_height)) = entry_receiver.recv().unwrap(); - assert!(e.is_tick()); + let WorkingBankEntry { + bank: _, + entries_ticks, + } = entry_receiver.recv().unwrap(); + assert_eq!(entries_ticks.len(), 1); + assert!(entries_ticks[0].0.is_tick()); } - let (_bank, (e, _tick_height)) = entry_receiver.recv().unwrap(); - assert!(!e.is_tick()); + let WorkingBankEntry { + bank: _, + entries_ticks, + } = entry_receiver.recv().unwrap(); + assert_eq!(entries_ticks.len(), 1); + assert!(!entries_ticks[0].0.is_tick()); } #[test] @@ -1560,11 +1602,15 @@ mod tests { let tx = test_tx(); let h1 = hash(b"hello world!"); assert!(poh_recorder - .record(bank.slot(), h1, vec![tx.into()]) + .record(bank.slot(), &[(h1, vec![tx.into()])]) .is_err()); for _ in 0..num_ticks_to_max { - let (_bank, (entry, _tick_height)) = entry_receiver.recv().unwrap(); - assert!(entry.is_tick()); + let WorkingBankEntry { + bank: _, + entries_ticks, + } = entry_receiver.recv().unwrap(); + assert_eq!(entries_ticks.len(), 1); + assert!(entries_ticks[0].0.is_tick()); } } @@ -1604,7 +1650,7 @@ mod tests { let tx1 = test_tx(); let h1 = hash(b"hello world!"); let record_result = poh_recorder - .record(bank.slot(), h1, vec![tx0.into(), tx1.into()]) + .record(bank.slot(), &[(h1, vec![tx0.into(), tx1.into()])]) .unwrap() .unwrap(); assert_eq!(record_result, 0); @@ -1621,7 +1667,7 @@ mod tests { let tx = test_tx(); let h2 = hash(b"foobar"); let record_result = poh_recorder - .record(bank.slot(), h2, vec![tx.into()]) + .record(bank.slot(), &[(h2, vec![tx.into()])]) .unwrap() .unwrap(); assert_eq!(record_result, 2); @@ -1860,7 +1906,7 @@ mod tests { let tx = test_tx(); let h1 = hash(b"hello world!"); assert!(poh_recorder - .record(bank.slot(), h1, vec![tx.into()]) + .record(bank.slot(), &[(h1, vec![tx.into()])]) .is_err()); assert!(poh_recorder.working_bank.is_none()); diff --git a/poh/src/poh_service.rs b/poh/src/poh_service.rs index 04dc8d4c5a..e5b40d6577 100644 --- a/poh/src/poh_service.rs +++ b/poh/src/poh_service.rs @@ -192,11 +192,12 @@ impl PohService { if let Ok(record) = record { if record .sender - .send(poh_recorder.write().unwrap().record( - record.slot, - record.mixin, - record.transactions, - )) + .send( + poh_recorder + .write() + .unwrap() + .record(record.slot, &record.mixins_txs), + ) .is_err() { panic!("Error returning mixin hash"); @@ -255,11 +256,7 @@ impl PohService { timing.total_lock_time_ns += lock_time.as_ns(); let mut record_time = Measure::start("record"); loop { - let res = poh_recorder_l.record( - record.slot, - record.mixin, - std::mem::take(&mut record.transactions), - ); + let res = poh_recorder_l.record(record.slot, &record.mixins_txs); // what do we do on failure here? Ignore for now. let (_send_res, send_record_result_time) = measure!(record.sender.send(res), "send_record_result"); @@ -381,6 +378,7 @@ impl PohService { mod tests { use { super::*, + crate::poh_recorder::WorkingBankEntry, rand::{thread_rng, Rng}, solana_ledger::{ blockstore::Blockstore, @@ -456,11 +454,10 @@ mod tests { loop { // send some data let mut time = Measure::start("record"); - let _ = - poh_recorder - .write() - .unwrap() - .record(bank_slot, h1, vec![tx.clone()]); + let _ = poh_recorder + .write() + .unwrap() + .record(bank_slot, &[(h1, vec![tx.clone()])]); time.stop(); total_us += time.as_us(); total_times += 1; @@ -505,7 +502,12 @@ mod tests { let time = Instant::now(); while run_time != 0 || need_tick || need_entry || need_partial { - let (_bank, (entry, _tick_height)) = entry_receiver.recv().unwrap(); + let WorkingBankEntry { + bank: _, + mut entries_ticks, + } = entry_receiver.recv().unwrap(); + assert_eq!(entries_ticks.len(), 1); + let entry = entries_ticks.pop().unwrap().0; if entry.is_tick() { num_ticks += 1; diff --git a/program-test/src/programs.rs b/program-test/src/programs.rs index e839b2c090..95fa3d55d0 100644 --- a/program-test/src/programs.rs +++ b/program-test/src/programs.rs @@ -12,6 +12,14 @@ mod spl_memo_1_0 { mod spl_memo_3_0 { solana_sdk::declare_id!("MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"); } + +mod jito_tip_payment { + solana_sdk::declare_id!("T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt"); +} +mod jito_tip_distribution { + solana_sdk::declare_id!("4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7"); +} + static SPL_PROGRAMS: &[(Pubkey, Pubkey, &[u8])] = &[ ( solana_inline_spl::token::ID, @@ -38,6 +46,16 @@ static SPL_PROGRAMS: &[(Pubkey, Pubkey, &[u8])] = &[ solana_sdk::bpf_loader::ID, include_bytes!("programs/spl_associated_token_account-1.1.1.so"), ), + ( + jito_tip_distribution::ID, + solana_sdk::bpf_loader::ID, + include_bytes!("programs/jito_tip_distribution-0.1.4.so"), + ), + ( + jito_tip_payment::ID, + solana_sdk::bpf_loader::ID, + include_bytes!("programs/jito_tip_payment-0.1.4.so"), + ), ]; /// Returns a tuple `(Pubkey, Account)` for a BPF program, where the key is the diff --git a/program-test/src/programs/jito_tip_distribution-0.1.4.so b/program-test/src/programs/jito_tip_distribution-0.1.4.so new file mode 100644 index 0000000000..29fc1a0eb5 Binary files /dev/null and b/program-test/src/programs/jito_tip_distribution-0.1.4.so differ diff --git a/program-test/src/programs/jito_tip_payment-0.1.4.so b/program-test/src/programs/jito_tip_payment-0.1.4.so new file mode 100644 index 0000000000..49e574ea8c Binary files /dev/null and b/program-test/src/programs/jito_tip_payment-0.1.4.so differ diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index f2213a8e92..9a2c09592e 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -123,6 +123,7 @@ dependencies = [ "solana-rpc-client", "solana-rpc-client-api", "solana-runtime", + "solana-runtime-plugin", "solana-sdk", "solana-send-transaction-service", "solana-storage-bigtable", @@ -136,6 +137,7 @@ dependencies = [ "thiserror", "tikv-jemallocator", "tokio", + "tonic", ] [[package]] @@ -201,6 +203,145 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "anchor-attribute-access-control" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "regex", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-account" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "bs58 0.4.0", + "proc-macro2 1.0.86", + "quote 1.0.36", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-constant" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "proc-macro2 1.0.86", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-error" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-event" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-interface" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "heck 0.3.3", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-program" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-attribute-state" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-derive-accounts" +version = "0.24.2" +dependencies = [ + "anchor-syn", + "anyhow", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + +[[package]] +name = "anchor-lang" +version = "0.24.2" +dependencies = [ + "anchor-attribute-access-control", + "anchor-attribute-account", + "anchor-attribute-constant", + "anchor-attribute-error", + "anchor-attribute-event", + "anchor-attribute-interface", + "anchor-attribute-program", + "anchor-attribute-state", + "anchor-derive-accounts", + "arrayref", + "base64 0.13.1", + "bincode", + "borsh 0.10.3", + "bytemuck", + "solana-program", + "thiserror", +] + +[[package]] +name = "anchor-syn" +version = "0.24.2" +dependencies = [ + "anyhow", + "bs58 0.3.1", + "heck 0.3.3", + "proc-macro2 1.0.86", + "proc-macro2-diagnostics", + "quote 1.0.36", + "serde", + "serde_json", + "sha2 0.9.9", + "syn 1.0.109", + "thiserror", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -240,8 +381,8 @@ dependencies = [ "include_dir", "itertools 0.10.5", "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -305,7 +446,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" dependencies = [ - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -317,8 +458,8 @@ checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ "num-bigint 0.4.6", "num-traits", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -353,8 +494,8 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae3281bc6d0fd7e549af32b52511e1302185bd688fd3359fa36423346ff682ea" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -414,8 +555,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "synstructure", ] @@ -426,8 +567,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -488,8 +629,8 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -499,8 +640,8 @@ version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -649,8 +790,8 @@ dependencies = [ "itertools 0.12.1", "lazy_static", "lazycell", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "regex", "rustc-hash", "shlex", @@ -749,7 +890,7 @@ dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", - "proc-macro2", + "proc-macro2 1.0.86", "syn 1.0.109", ] @@ -761,8 +902,8 @@ checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" dependencies = [ "once_cell", "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", "syn_derive", ] @@ -773,8 +914,8 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -784,8 +925,8 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -810,6 +951,18 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb" + +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + [[package]] name = "bs58" version = "0.5.1" @@ -860,8 +1013,8 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -1265,8 +1418,8 @@ checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "strsim 0.10.0", "syn 2.0.58", ] @@ -1278,7 +1431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core", - "quote", + "quote 1.0.36", "syn 2.0.58", ] @@ -1302,6 +1455,17 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "default-env" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f753eb82d29277e79efc625e84aecacfd4851ee50e05a8573a4740239a77bfd3" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "der-parser" version = "8.2.0" @@ -1337,8 +1501,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1349,8 +1513,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustc_version", "syn 1.0.109", ] @@ -1429,8 +1593,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -1452,8 +1616,8 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cbae11b3de8fce2a456e8ea3dada226b35fe791f0dc1d360c0941f0bb681f3" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -1511,8 +1675,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f0042ff8246a363dbe77d2ceedb073339e85a804b9a47636c6e016a9a32c05f" dependencies = [ "enum-ordinalize", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -1561,8 +1725,8 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -1574,8 +1738,8 @@ checksum = "1bf1fa3f06bbff1ea5b1a9c7b14aa992a39657db60a2759457328d7e058f49ee" dependencies = [ "num-bigint 0.4.6", "num-traits", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -1802,8 +1966,8 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -2008,6 +2172,15 @@ dependencies = [ "http", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.0" @@ -2263,8 +2436,8 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b139284b5cf57ecfa712bcc66950bb635b31aff41c188e8a4cfc758eca374a3f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", ] [[package]] @@ -2355,6 +2528,49 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jito-programs-vote-state" +version = "0.1.5" +dependencies = [ + "anchor-lang", + "bincode", + "serde", + "serde_derive", + "solana-program", +] + +[[package]] +name = "jito-protos" +version = "2.1.0" +dependencies = [ + "bytes", + "prost", + "prost-types", + "protobuf-src", + "tonic", + "tonic-build", +] + +[[package]] +name = "jito-tip-distribution" +version = "0.1.5" +dependencies = [ + "anchor-lang", + "default-env", + "jito-programs-vote-state", + "solana-program", + "solana-security-txt", +] + +[[package]] +name = "jito-tip-payment" +version = "0.1.5" +dependencies = [ + "anchor-lang", + "default-env", + "solana-security-txt", +] + [[package]] name = "jobserver" version = "0.1.21" @@ -2435,8 +2651,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" dependencies = [ "proc-macro-crate 0.1.5", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2894,8 +3110,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ "cfg-if 1.0.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -2915,8 +3131,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3040,8 +3256,8 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -3112,8 +3328,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -3174,8 +3390,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -3377,8 +3593,8 @@ checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" dependencies = [ "pest", "pest_meta", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -3418,8 +3634,8 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -3513,7 +3729,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b83ec2d0af5c5c556257ff52c9f98934e243b9fd39604bfb2a9b75ec2e97f18" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.86", "syn 1.0.109", ] @@ -3548,8 +3764,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", "version_check", ] @@ -3560,11 +3776,20 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "version_check", ] +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -3574,6 +3799,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada" +dependencies = [ + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", + "version_check", + "yansi", +] + [[package]] name = "prost" version = "0.11.9" @@ -3591,7 +3829,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", - "heck", + "heck 0.4.0", "itertools 0.10.5", "lazy_static", "log", @@ -3614,8 +3852,8 @@ checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -3652,8 +3890,8 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -3705,13 +3943,22 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.86", ] [[package]] @@ -4074,7 +4321,7 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.3", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] @@ -4099,6 +4346,16 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "rustls-webpki" +version = "0.100.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" +dependencies = [ + "ring 0.16.20", + "untrusted 0.7.1", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4223,8 +4480,8 @@ version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -4268,8 +4525,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" dependencies = [ "darling", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -4494,7 +4751,7 @@ dependencies = [ "Inflector", "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "bv", "lazy_static", "serde", @@ -4610,6 +4867,7 @@ dependencies = [ "futures 0.3.30", "solana-banks-interface", "solana-client", + "solana-gossip", "solana-runtime", "solana-sdk", "solana-send-transaction-service", @@ -4687,6 +4945,28 @@ dependencies = [ "tempfile", ] +[[package]] +name = "solana-bundle" +version = "2.1.0" +dependencies = [ + "anchor-lang", + "itertools 0.12.1", + "log", + "serde", + "solana-accounts-db", + "solana-ledger", + "solana-logger", + "solana-measure", + "solana-poh", + "solana-program-runtime", + "solana-runtime", + "solana-sdk", + "solana-svm", + "solana-timings", + "solana-transaction-status", + "thiserror", +] + [[package]] name = "solana-clap-utils" version = "2.1.0" @@ -4826,10 +5106,11 @@ name = "solana-core" version = "2.1.0" dependencies = [ "ahash 0.8.10", + "anchor-lang", "arrayvec", "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "bytes", "chrono", "crossbeam-channel", @@ -4838,12 +5119,17 @@ dependencies = [ "futures 0.3.30", "histogram", "itertools 0.12.1", + "jito-protos", + "jito-tip-distribution", + "jito-tip-payment", "lazy_static", "log", "lru", "min-max-heap", "num_enum", "prio-graph", + "prost", + "prost-types", "qualifier_attr", "quinn", "rand 0.8.5", @@ -4857,6 +5143,7 @@ dependencies = [ "serde_derive", "solana-accounts-db", "solana-bloom", + "solana-bundle", "solana-client", "solana-compute-budget", "solana-connection-cache", @@ -4875,6 +5162,7 @@ dependencies = [ "solana-rpc", "solana-rpc-client-api", "solana-runtime", + "solana-runtime-plugin", "solana-sanitize", "solana-sdk", "solana-send-transaction-service", @@ -4897,6 +5185,8 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tonic", + "tonic-build", "trees", ] @@ -5013,7 +5303,7 @@ name = "solana-geyser-plugin-manager" version = "2.1.0" dependencies = [ "agave-geyser-plugin-interface", - "bs58", + "bs58 0.5.1", "crossbeam-channel", "json5", "jsonrpc-core", @@ -5304,7 +5594,7 @@ dependencies = [ "blake3", "borsh 0.10.3", "borsh 1.5.1", - "bs58", + "bs58 0.5.1", "bv", "bytemuck", "bytemuck_derive", @@ -5476,7 +5766,7 @@ version = "2.1.0" dependencies = [ "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "crossbeam-channel", "dashmap", "itertools 0.12.1", @@ -5495,6 +5785,7 @@ dependencies = [ "soketto", "solana-account-decoder", "solana-accounts-db", + "solana-bundle", "solana-client", "solana-entry", "solana-faucet", @@ -5505,6 +5796,7 @@ dependencies = [ "solana-metrics", "solana-perf", "solana-poh", + "solana-program-runtime", "solana-rayon-threadlimit", "solana-rpc-client-api", "solana-runtime", @@ -5534,7 +5826,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bincode", - "bs58", + "bs58 0.5.1", "indicatif", "log", "reqwest", @@ -5558,7 +5850,7 @@ version = "2.1.0" dependencies = [ "anyhow", "base64 0.22.1", - "bs58", + "bs58 0.5.1", "jsonrpc-core", "reqwest", "reqwest-middleware", @@ -5567,8 +5859,10 @@ dependencies = [ "serde_derive", "serde_json", "solana-account-decoder", + "solana-bundle", "solana-inline-spl", "solana-sdk", + "solana-svm", "solana-transaction-status", "solana-version", "thiserror", @@ -5664,6 +5958,24 @@ dependencies = [ "zstd", ] +[[package]] +name = "solana-runtime-plugin" +version = "2.1.0" +dependencies = [ + "crossbeam-channel", + "json5", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "libloading 0.7.4", + "log", + "solana-runtime", + "solana-sdk", + "thiserror", +] + [[package]] name = "solana-sanitize" version = "2.1.0" @@ -6137,10 +6449,12 @@ dependencies = [ name = "solana-sdk" version = "2.1.0" dependencies = [ + "anchor-lang", + "base64 0.22.1", "bincode", "bitflags 2.6.0", "borsh 1.5.1", - "bs58", + "bs58 0.5.1", "bytemuck", "bytemuck_derive", "byteorder 1.5.0", @@ -6189,9 +6503,9 @@ dependencies = [ name = "solana-sdk-macro" version = "2.1.0" dependencies = [ - "bs58", - "proc-macro2", - "quote", + "bs58 0.5.1", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -6220,6 +6534,7 @@ dependencies = [ "log", "solana-client", "solana-connection-cache", + "solana-gossip", "solana-measure", "solana-metrics", "solana-runtime", @@ -6287,7 +6602,7 @@ name = "solana-storage-proto" version = "2.1.0" dependencies = [ "bincode", - "bs58", + "bs58 0.5.1", "prost", "protobuf-src", "serde", @@ -6464,7 +6779,7 @@ dependencies = [ "base64 0.22.1", "bincode", "borsh 1.5.1", - "bs58", + "bs58 0.5.1", "lazy_static", "log", "serde", @@ -6776,7 +7091,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9e8418ea6269dcfb01c712f0444d2c75542c04448b480e87de59d2865edc750" dependencies = [ - "quote", + "quote 1.0.36", "spl-discriminator-syn", "syn 2.0.58", ] @@ -6787,8 +7102,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c1f05593b7ca9eac7caca309720f2eafb96355e037e6d373b909a80fe7b69b9" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "sha2 0.10.8", "syn 2.0.58", "thiserror", @@ -6836,8 +7151,8 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d375dd76c517836353e093c2dbb490938ff72821ab568b545fd30ab3256b3e" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "sha2 0.10.8", "syn 2.0.58", ] @@ -6995,9 +7310,9 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck", - "proc-macro2", - "quote", + "heck 0.4.0", + "proc-macro2 1.0.86", + "quote 1.0.36", "rustversion", "syn 1.0.109", ] @@ -7014,14 +7329,25 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "unicode-ident", ] @@ -7031,8 +7357,8 @@ version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "unicode-ident", ] @@ -7043,8 +7369,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" dependencies = [ "proc-macro-error", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -7060,10 +7386,10 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", - "unicode-xid", + "unicode-xid 0.2.4", ] [[package]] @@ -7151,8 +7477,8 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee42b4e559f17bce0385ebf511a7beb67d5cc33c12c96b7f4e9789919d9c10f" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 1.0.109", ] @@ -7222,8 +7548,8 @@ version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -7359,8 +7685,8 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -7503,6 +7829,7 @@ dependencies = [ "percent-encoding 2.3.1", "pin-project", "prost", + "rustls-native-certs", "rustls-pemfile", "tokio", "tokio-rustls", @@ -7511,6 +7838,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "webpki-roots 0.23.1", ] [[package]] @@ -7520,9 +7848,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" dependencies = [ "prettyplease", - "proc-macro2", + "proc-macro2 1.0.86", "prost-build", - "quote", + "quote 1.0.36", "syn 1.0.109", ] @@ -7577,8 +7905,8 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -7691,12 +8019,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + [[package]] name = "unicode-width" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.4" @@ -7866,8 +8206,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", "wasm-bindgen-shared", ] @@ -7890,7 +8230,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ - "quote", + "quote 1.0.36", "wasm-bindgen-macro-support", ] @@ -7900,8 +8240,8 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -7923,13 +8263,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" +dependencies = [ + "rustls-webpki 0.100.3", +] + [[package]] name = "webpki-roots" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki", + "rustls-webpki 0.101.7", ] [[package]] @@ -8172,6 +8521,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zerocopy" version = "0.7.31" @@ -8187,8 +8542,8 @@ version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] @@ -8207,8 +8562,8 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.86", + "quote 1.0.36", "syn 2.0.58", ] diff --git a/programs/sbf/tests/programs.rs b/programs/sbf/tests/programs.rs index 99cdba5666..fff202e076 100644 --- a/programs/sbf/tests/programs.rs +++ b/programs/sbf/tests/programs.rs @@ -137,7 +137,7 @@ fn execute_transactions( let batch = bank.prepare_batch_for_tests(txs.clone()); let mut timings = ExecuteTimings::default(); let mut mint_decimals = HashMap::new(); - let tx_pre_token_balances = collect_token_balances(&bank, &batch, &mut mint_decimals); + let tx_pre_token_balances = collect_token_balances(&bank, &batch, &mut mint_decimals, None); let ( TransactionResults { execution_results, .. @@ -155,7 +155,7 @@ fn execute_transactions( &mut timings, None, ); - let tx_post_token_balances = collect_token_balances(&bank, &batch, &mut mint_decimals); + let tx_post_token_balances = collect_token_balances(&bank, &batch, &mut mint_decimals, None); izip!( txs.iter(), diff --git a/rpc-client-api/Cargo.toml b/rpc-client-api/Cargo.toml index 22a883244c..bfc61c9ba1 100644 --- a/rpc-client-api/Cargo.toml +++ b/rpc-client-api/Cargo.toml @@ -21,8 +21,10 @@ serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } solana-account-decoder = { workspace = true } +solana-bundle = { workspace = true } solana-inline-spl = { workspace = true } solana-sdk = { workspace = true } +solana-svm = { workspace = true } solana-transaction-status = { workspace = true } solana-version = { workspace = true } thiserror = { workspace = true } diff --git a/rpc-client-api/src/bundles.rs b/rpc-client-api/src/bundles.rs new file mode 100644 index 0000000000..5f47a102e9 --- /dev/null +++ b/rpc-client-api/src/bundles.rs @@ -0,0 +1,166 @@ +//! Use a separate file for Jito related code to minimize upstream merge conflicts. + +use { + crate::config::RpcSimulateTransactionAccountsConfig, + solana_account_decoder::UiAccount, + solana_bundle::{bundle_execution::LoadAndExecuteBundleError, BundleExecutionError}, + solana_sdk::{ + clock::Slot, + commitment_config::{CommitmentConfig, CommitmentLevel}, + signature::Signature, + transaction::TransactionError, + }, + solana_svm::transaction_results::TransactionExecutionResult, + solana_transaction_status::{UiTransactionEncoding, UiTransactionReturnData}, + thiserror::Error, +}; + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub enum RpcBundleSimulationSummary { + /// error and offending transaction signature if applicable + Failed { + error: RpcBundleExecutionError, + tx_signature: Option, + }, + Succeeded, +} + +#[derive(Error, Debug, Clone, Serialize, Deserialize)] +pub enum RpcBundleExecutionError { + #[error("The bank has hit the max allotted time for processing transactions")] + BankProcessingTimeLimitReached, + + #[error("Error locking bundle because a transaction is malformed")] + BundleLockError, + + #[error("Bundle execution timed out")] + BundleExecutionTimeout, + + #[error("The bundle exceeds the cost model")] + ExceedsCostModel, + + #[error("Invalid pre or post accounts")] + InvalidPreOrPostAccounts, + + #[error("PoH record error: {0}")] + PohRecordError(String), + + #[error("Tip payment error: {0}")] + TipError(String), + + #[error("A transaction in the bundle failed to execute: [signature={0}, error={1}]")] + TransactionFailure(Signature, String), +} + +impl From for RpcBundleExecutionError { + fn from(bundle_execution_error: BundleExecutionError) -> Self { + match bundle_execution_error { + BundleExecutionError::BankProcessingTimeLimitReached => { + Self::BankProcessingTimeLimitReached + } + BundleExecutionError::ExceedsCostModel => Self::ExceedsCostModel, + BundleExecutionError::TransactionFailure(load_and_execute_bundle_error) => { + match load_and_execute_bundle_error { + LoadAndExecuteBundleError::ProcessingTimeExceeded(_) => { + Self::BundleExecutionTimeout + } + LoadAndExecuteBundleError::LockError { + signature, + transaction_error, + } => Self::TransactionFailure(signature, transaction_error.to_string()), + LoadAndExecuteBundleError::TransactionError { + signature, + execution_result, + } => match *execution_result { + TransactionExecutionResult::Executed { details, .. } => { + let err_msg = if let Err(e) = details.status { + e.to_string() + } else { + "Unknown error".to_string() + }; + Self::TransactionFailure(signature, err_msg) + } + TransactionExecutionResult::NotExecuted(e) => { + Self::TransactionFailure(signature, e.to_string()) + } + }, + LoadAndExecuteBundleError::InvalidPreOrPostAccounts => { + Self::InvalidPreOrPostAccounts + } + } + } + BundleExecutionError::LockError => Self::BundleLockError, + BundleExecutionError::PohRecordError(e) => Self::PohRecordError(e.to_string()), + BundleExecutionError::TipError(e) => Self::TipError(e.to_string()), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct RpcSimulateBundleResult { + pub summary: RpcBundleSimulationSummary, + pub transaction_results: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct RpcSimulateBundleTransactionResult { + pub err: Option, + pub logs: Option>, + pub pre_execution_accounts: Option>, + pub post_execution_accounts: Option>, + pub units_consumed: Option, + pub return_data: Option, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcSimulateBundleConfig { + /// Gives the state of accounts pre/post transaction execution. + /// The length of each of these must be equal to the number transactions. + pub pre_execution_accounts_configs: Vec>, + pub post_execution_accounts_configs: Vec>, + + /// Specifies the encoding scheme of the contained transactions. + pub transaction_encoding: Option, + + /// Specifies the bank to run simulation against. + pub simulation_bank: Option, + + /// Opt to skip sig-verify for faster performance. + #[serde(default)] + pub skip_sig_verify: bool, + + /// Replace recent blockhash to simulate old transactions without resigning. + #[serde(default)] + pub replace_recent_blockhash: bool, +} + +#[derive(Serialize, Deserialize, Clone, Copy, Debug)] +#[serde(rename_all = "camelCase")] +pub enum SimulationSlotConfig { + /// Simulate on top of bank with the provided commitment. + Commitment(CommitmentConfig), + + /// Simulate on the provided slot's bank. + Slot(Slot), + + /// Simulates on top of the RPC's highest slot's bank i.e. the working bank. + Tip, +} + +impl Default for SimulationSlotConfig { + fn default() -> Self { + Self::Commitment(CommitmentConfig { + commitment: CommitmentLevel::Confirmed, + }) + } +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcBundleRequest { + pub encoded_transactions: Vec, +} diff --git a/rpc-client-api/src/lib.rs b/rpc-client-api/src/lib.rs index b248463776..198442c2b7 100644 --- a/rpc-client-api/src/lib.rs +++ b/rpc-client-api/src/lib.rs @@ -1,5 +1,6 @@ #![allow(clippy::arithmetic_side_effects)] +pub mod bundles; pub mod client_error; pub mod config; pub mod custom_error; diff --git a/rpc-client-api/src/request.rs b/rpc-client-api/src/request.rs index fe032a858d..5b0547c90e 100644 --- a/rpc-client-api/src/request.rs +++ b/rpc-client-api/src/request.rs @@ -66,6 +66,7 @@ pub enum RpcRequest { RequestAirdrop, SendTransaction, SimulateTransaction, + SimulateBundle, SignVote, } @@ -131,6 +132,7 @@ impl fmt::Display for RpcRequest { RpcRequest::RequestAirdrop => "requestAirdrop", RpcRequest::SendTransaction => "sendTransaction", RpcRequest::SimulateTransaction => "simulateTransaction", + RpcRequest::SimulateBundle => "simulateBundle", RpcRequest::SignVote => "signVote", }; diff --git a/rpc-client/src/nonblocking/rpc_client.rs b/rpc-client/src/nonblocking/rpc_client.rs index 0ca5f76a49..153332b65a 100644 --- a/rpc-client/src/nonblocking/rpc_client.rs +++ b/rpc-client/src/nonblocking/rpc_client.rs @@ -28,6 +28,10 @@ use { UiAccount, UiAccountData, UiAccountEncoding, }, solana_rpc_client_api::{ + bundles::{ + RpcBundleRequest, RpcSimulateBundleConfig, RpcSimulateBundleResult, + SimulationSlotConfig, + }, client_error::{ Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult, }, @@ -37,6 +41,7 @@ use { }, solana_sdk::{ account::Account, + bundle::VersionedBundle, clock::{Epoch, Slot, UnixTimestamp, DEFAULT_MS_PER_SLOT}, commitment_config::CommitmentConfig, epoch_info::EpochInfo, @@ -44,7 +49,7 @@ use { hash::Hash, pubkey::Pubkey, signature::Signature, - transaction, + transaction::{self, VersionedTransaction}, }, solana_transaction_status::{ EncodedConfirmedBlock, EncodedConfirmedTransactionWithStatusMeta, TransactionStatus, @@ -885,6 +890,7 @@ impl RpcClient { code, message, data, + .. }) = &err.kind { debug!("{} {}", code, message); @@ -1312,6 +1318,54 @@ impl RpcClient { .await } + pub async fn simulate_bundle( + &self, + bundle: &VersionedBundle, + ) -> RpcResult { + self.simulate_bundle_with_config( + bundle, + RpcSimulateBundleConfig { + simulation_bank: Some(SimulationSlotConfig::Commitment(self.commitment())), + pre_execution_accounts_configs: vec![None; bundle.transactions.len()], + post_execution_accounts_configs: vec![None; bundle.transactions.len()], + ..RpcSimulateBundleConfig::default() + }, + ) + .await + } + + pub async fn simulate_bundle_with_config( + &self, + bundle: &VersionedBundle, + config: RpcSimulateBundleConfig, + ) -> RpcResult { + let transaction_encoding = config + .transaction_encoding + .unwrap_or(UiTransactionEncoding::Base64); + let simulation_bank = Some(config.simulation_bank.unwrap_or_default()); + + let encoded_transactions = bundle + .transactions + .iter() + .map(|tx| serialize_and_encode::(tx, transaction_encoding)) + .collect::>>()?; + let rpc_bundle_request = RpcBundleRequest { + encoded_transactions, + }; + + let config = RpcSimulateBundleConfig { + transaction_encoding: Some(transaction_encoding), + simulation_bank, + ..config + }; + + self.send( + RpcRequest::SimulateBundle, + json!([rpc_bundle_request, config]), + ) + .await + } + /// Returns the highest slot information that the node has snapshots for. /// /// This will find the highest full snapshot slot, and the highest incremental snapshot slot diff --git a/rpc-client/src/rpc_client.rs b/rpc-client/src/rpc_client.rs index 32bd08cef4..e4167d6330 100644 --- a/rpc-client/src/rpc_client.rs +++ b/rpc-client/src/rpc_client.rs @@ -24,6 +24,7 @@ use { UiAccount, UiAccountEncoding, }, solana_rpc_client_api::{ + bundles::{RpcSimulateBundleConfig, RpcSimulateBundleResult}, client_error::{Error as ClientError, ErrorKind, Result as ClientResult}, config::{RpcAccountInfoConfig, *}, request::{RpcRequest, TokenAccountsFilter}, @@ -31,6 +32,7 @@ use { }, solana_sdk::{ account::{Account, ReadableAccount}, + bundle::VersionedBundle, clock::{Epoch, Slot, UnixTimestamp}, commitment_config::CommitmentConfig, epoch_info::EpochInfo, @@ -1146,6 +1148,18 @@ impl RpcClient { ) } + pub fn simulate_bundle(&self, bundle: &VersionedBundle) -> RpcResult { + self.invoke((self.rpc_client.as_ref()).simulate_bundle(bundle)) + } + + pub fn simulate_bundle_with_config( + &self, + bundle: &VersionedBundle, + config: RpcSimulateBundleConfig, + ) -> RpcResult { + self.invoke((self.rpc_client.as_ref()).simulate_bundle_with_config(bundle, config)) + } + /// Returns the highest slot information that the node has snapshots for. /// /// This will find the highest full snapshot slot, and the highest incremental snapshot slot diff --git a/rpc-test/Cargo.toml b/rpc-test/Cargo.toml index 435edafdc5..f9bec675ea 100644 --- a/rpc-test/Cargo.toml +++ b/rpc-test/Cargo.toml @@ -33,6 +33,7 @@ solana-transaction-status = { workspace = true } tokio = { workspace = true, features = ["full"] } [dev-dependencies] +serial_test = { workspace = true } solana-connection-cache = { workspace = true } solana-logger = { workspace = true } diff --git a/rpc-test/tests/rpc.rs b/rpc-test/tests/rpc.rs index 67d229d43a..51855bc5e5 100644 --- a/rpc-test/tests/rpc.rs +++ b/rpc-test/tests/rpc.rs @@ -5,6 +5,7 @@ use { log::*, reqwest::{self, header::CONTENT_TYPE}, serde_json::{json, Value}, + serial_test::serial, solana_account_decoder::UiAccount, solana_client::connection_cache::ConnectionCache, solana_pubsub_client::nonblocking::pubsub_client::PubsubClient, @@ -283,6 +284,7 @@ fn test_rpc_slot_updates() { } #[test] +#[serial] // helps test pass fn test_rpc_subscriptions() { solana_logger::setup(); diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 2a0c5c480d..b9616c2fb0 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -31,6 +31,7 @@ serde_json = { workspace = true } soketto = { workspace = true } solana-account-decoder = { workspace = true } solana-accounts-db = { workspace = true } +solana-bundle = { workspace = true } solana-client = { workspace = true } solana-entry = { workspace = true } solana-faucet = { workspace = true } @@ -41,6 +42,7 @@ solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-perf = { workspace = true } solana-poh = { workspace = true } +solana-program-runtime = { workspace = true } solana-rayon-threadlimit = { workspace = true } solana-rpc-client-api = { workspace = true } solana-runtime = { workspace = true } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index d62a61ec81..43c4e483a7 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -61,7 +61,7 @@ use { }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, - clock::{Slot, UnixTimestamp, MAX_PROCESSING_AGE}, + clock::{Slot, UnixTimestamp}, commitment_config::{CommitmentConfig, CommitmentLevel}, epoch_info::EpochInfo, epoch_rewards_hasher::EpochRewardsHasher, @@ -242,6 +242,13 @@ impl JsonRpcRequestProcessor { Ok(bank) } + fn bank_from_slot(&self, slot: Slot) -> Option> { + debug!("Slot: {:?}", slot); + + let r_bank_forks = self.bank_forks.read().unwrap(); + r_bank_forks.get(slot) + } + #[allow(deprecated)] fn bank(&self, commitment: Option) -> Arc { debug!("RPC commitment_config: {:?}", commitment); @@ -364,13 +371,10 @@ impl JsonRpcRequestProcessor { ); ClusterInfo::new(contact_info, keypair, socket_addr_space) }); - let tpu_address = cluster_info - .my_contact_info() - .tpu(connection_cache.protocol()) - .unwrap(); + let (sender, receiver) = unbounded(); SendTransactionService::new::( - tpu_address, + cluster_info.clone(), &bank_forks, None, receiver, @@ -2687,13 +2691,16 @@ pub mod rpc_minimal { }) .unwrap(); - let full_snapshot_slot = - snapshot_utils::get_highest_full_snapshot_archive_slot(full_snapshot_archives_dir) - .ok_or(RpcCustomError::NoSnapshot)?; + let full_snapshot_slot = snapshot_utils::get_highest_full_snapshot_archive_slot( + full_snapshot_archives_dir, + None, + ) + .ok_or(RpcCustomError::NoSnapshot)?; let incremental_snapshot_slot = snapshot_utils::get_highest_incremental_snapshot_archive_slot( incremental_snapshot_archives_dir, full_snapshot_slot, + None, ); Ok(RpcSnapshotSlotInfo { @@ -3278,14 +3285,169 @@ pub mod rpc_accounts_scan { } } +pub mod utils { + use { + crate::rpc::encode_account, + jsonrpc_core::Error, + solana_account_decoder::{UiAccount, UiAccountEncoding}, + solana_bundle::{ + bundle_execution::{LoadAndExecuteBundleError, LoadAndExecuteBundleOutput}, + BundleExecutionError, + }, + solana_rpc_client_api::{ + bundles::{ + RpcBundleExecutionError, RpcBundleSimulationSummary, RpcSimulateBundleConfig, + RpcSimulateBundleResult, RpcSimulateBundleTransactionResult, + }, + config::RpcSimulateTransactionAccountsConfig, + }, + solana_sdk::{account::AccountSharedData, pubkey::Pubkey}, + std::str::FromStr, + }; + + /// Encodes the accounts, returns an error if any of the accounts failed to encode + /// The outer error can be set by error parsing, Ok(None) means there wasn't any accounts in the parameter + fn try_encode_accounts( + accounts: &Option>, + encoding: UiAccountEncoding, + ) -> Result>, Error> { + if let Some(accounts) = accounts { + Ok(Some( + accounts + .iter() + .map(|(pubkey, account)| encode_account(account, pubkey, encoding, None)) + .collect::, Error>>()?, + )) + } else { + Ok(None) + } + } + + pub fn rpc_bundle_result_from_bank_result( + bundle_execution_result: LoadAndExecuteBundleOutput, + rpc_config: RpcSimulateBundleConfig, + ) -> Result { + let summary = match bundle_execution_result.result() { + Ok(_) => RpcBundleSimulationSummary::Succeeded, + Err(e) => { + let tx_signature = match e { + LoadAndExecuteBundleError::TransactionError { signature, .. } + | LoadAndExecuteBundleError::LockError { signature, .. } => { + Some(signature.to_string()) + } + _ => None, + }; + RpcBundleSimulationSummary::Failed { + error: RpcBundleExecutionError::from(BundleExecutionError::TransactionFailure( + e.clone(), + )), + tx_signature, + } + } + }; + + let mut transaction_results = Vec::new(); + for bundle_output in bundle_execution_result.bundle_transaction_results() { + for (index, execution_result) in bundle_output + .execution_results() + .iter() + .enumerate() + .filter(|(_, result)| result.was_executed()) + { + // things are filtered by was_executed, so safe to unwrap here + let result = execution_result.flattened_result(); + let details = execution_result.details().unwrap(); + + let account_config = rpc_config + .pre_execution_accounts_configs + .get(transaction_results.len()) + .ok_or_else(|| Error::invalid_params("the length of pre_execution_accounts_configs must match the number of transactions"))?; + let account_encoding = account_config + .as_ref() + .and_then(|config| config.encoding) + .unwrap_or(UiAccountEncoding::Base64); + + let pre_execution_accounts = if let Some(pre_tx_accounts) = + bundle_output.pre_tx_execution_accounts().get(index) + { + try_encode_accounts(pre_tx_accounts, account_encoding)? + } else { + None + }; + + let post_execution_accounts = if let Some(post_tx_accounts) = + bundle_output.post_tx_execution_accounts().get(index) + { + try_encode_accounts(post_tx_accounts, account_encoding)? + } else { + None + }; + + transaction_results.push(RpcSimulateBundleTransactionResult { + err: match result { + Ok(_) => None, + Err(e) => Some(e), + }, + logs: details.log_messages.clone(), + pre_execution_accounts, + post_execution_accounts, + units_consumed: Some(details.executed_units), + return_data: details.return_data.clone().map(|data| data.into()), + }); + } + } + + Ok(RpcSimulateBundleResult { + summary, + transaction_results, + }) + } + + pub fn account_configs_to_accounts( + accounts_config: &[Option], + ) -> Result>>, Error> { + let mut execution_accounts = Vec::new(); + for account_config in accounts_config { + let accounts = match account_config { + None => None, + Some(account_config) => Some( + account_config + .addresses + .iter() + .map(|a| { + Pubkey::from_str(a).map_err(|_| { + Error::invalid_params(format!("invalid pubkey provided: {}", a)) + }) + }) + .collect::, Error>>()?, + ), + }; + execution_accounts.push(accounts); + } + Ok(execution_accounts) + } +} + // Full RPC interface that an API node is expected to provide // (rpc_minimal should also be provided by an API node) pub mod rpc_full { use { super::*, - solana_sdk::message::{SanitizedVersionedMessage, VersionedMessage}, + crate::rpc::utils::{account_configs_to_accounts, rpc_bundle_result_from_bank_result}, + jsonrpc_core::ErrorCode, + solana_bundle::bundle_execution::{load_and_execute_bundle, LoadAndExecuteBundleError}, + solana_rpc_client_api::bundles::{ + RpcBundleRequest, RpcSimulateBundleConfig, RpcSimulateBundleResult, + SimulationSlotConfig, + }, + solana_sdk::{ + bundle::{derive_bundle_id, SanitizedBundle}, + clock::MAX_PROCESSING_AGE, + message::{SanitizedVersionedMessage, VersionedMessage}, + }, solana_transaction_status::UiInnerInstructions, }; + #[rpc] pub trait Full { type Metadata; @@ -3347,6 +3509,14 @@ pub mod rpc_full { config: Option, ) -> Result>; + #[rpc(meta, name = "simulateBundle")] + fn simulate_bundle( + &self, + meta: Self::Metadata, + rpc_bundle_request: RpcBundleRequest, + config: Option, + ) -> Result>; + #[rpc(meta, name = "minimumLedgerSlot")] fn minimum_ledger_slot(&self, meta: Self::Metadata) -> Result; @@ -3891,6 +4061,145 @@ pub mod rpc_full { )) } + // TODO (LB): probably want to add a max transaction size and max account return size and max + // allowable simulation time + fn simulate_bundle( + &self, + meta: Self::Metadata, + rpc_bundle_request: RpcBundleRequest, + config: Option, + ) -> Result> { + const MAX_BUNDLE_SIMULATION_TIME: Duration = Duration::from_millis(500); + + debug!("simulate_bundle rpc request received"); + + let config = config.unwrap_or_else(|| RpcSimulateBundleConfig { + pre_execution_accounts_configs: vec![ + None; + rpc_bundle_request.encoded_transactions.len() + ], + post_execution_accounts_configs: vec![ + None; + rpc_bundle_request.encoded_transactions.len() + ], + ..RpcSimulateBundleConfig::default() + }); + + // Run some request validations + if !(config.pre_execution_accounts_configs.len() + == rpc_bundle_request.encoded_transactions.len() + && config.post_execution_accounts_configs.len() + == rpc_bundle_request.encoded_transactions.len()) + { + return Err(Error::invalid_params( + "pre/post_execution_accounts_configs must be equal in length to the number of transactions", + )); + } + + let bank = match config.simulation_bank.unwrap_or_default() { + SimulationSlotConfig::Commitment(commitment) => Ok(meta.bank(Some(commitment))), + SimulationSlotConfig::Slot(slot) => meta.bank_from_slot(slot).ok_or_else(|| { + Error::invalid_params(format!("bank not found for the provided slot: {}", slot)) + }), + SimulationSlotConfig::Tip => Ok(meta.bank_forks.read().unwrap().working_bank()), + }?; + + let tx_encoding = config + .transaction_encoding + .unwrap_or(UiTransactionEncoding::Base64); + let binary_encoding = tx_encoding.into_binary_encoding().ok_or_else(|| { + Error::invalid_params(format!( + "Unsupported encoding: {}. Supported encodings are: base58 & base64", + tx_encoding + )) + })?; + let mut decoded_transactions = rpc_bundle_request + .encoded_transactions + .into_iter() + .map(|encoded_tx| { + decode_and_deserialize::(encoded_tx, binary_encoding) + .map(|de| de.1) + }) + .collect::>>()?; + + if config.replace_recent_blockhash { + if !config.skip_sig_verify { + return Err(Error::invalid_params( + "sigVerify may not be used with replaceRecentBlockhash", + )); + } + decoded_transactions.iter_mut().for_each(|tx| { + tx.message.set_recent_blockhash(bank.last_blockhash()); + }); + } + + let bundle_id = derive_bundle_id(&decoded_transactions); + let sanitized_bundle = SanitizedBundle { + transactions: decoded_transactions + .into_iter() + .map(|tx| { + sanitize_transaction(tx, bank.as_ref(), bank.get_reserved_account_keys()) + }) + .collect::>>()?, + bundle_id, + }; + + if !config.skip_sig_verify { + for tx in &sanitized_bundle.transactions { + verify_transaction(tx, &bank.feature_set)?; + } + } + + let pre_execution_accounts = + account_configs_to_accounts(&config.pre_execution_accounts_configs)?; + let post_execution_accounts = + account_configs_to_accounts(&config.post_execution_accounts_configs)?; + + let bundle_execution_result = load_and_execute_bundle( + &bank, + &sanitized_bundle, + MAX_PROCESSING_AGE, + &MAX_BUNDLE_SIMULATION_TIME, + true, + &None, + true, + None, + &pre_execution_accounts, + &post_execution_accounts, + ); + + // only return error if irrecoverable (timeout or tx malformed) + // bundle execution failures w/ context are returned to client + match bundle_execution_result.result() { + Ok(()) | Err(LoadAndExecuteBundleError::TransactionError { .. }) => {} + Err(LoadAndExecuteBundleError::ProcessingTimeExceeded(elapsed)) => { + let mut error = Error::new(ErrorCode::ServerError(10_000)); + error.message = format!( + "simulation time exceeded max allowed time: {:?}ms", + elapsed.as_millis() + ); + return Err(error); + } + Err(LoadAndExecuteBundleError::InvalidPreOrPostAccounts) => { + return Err(Error::invalid_params("invalid pre or post account data")); + } + Err(LoadAndExecuteBundleError::LockError { + signature, + transaction_error, + }) => { + return Err(Error::invalid_params(format!( + "error locking transaction with signature: {}, error: {:?}", + signature, transaction_error + ))); + } + } + + let rpc_bundle_result = + rpc_bundle_result_from_bank_result(bundle_execution_result, config)?; + + Ok(new_response(&bank, rpc_bundle_result)) + } + fn minimum_ledger_slot(&self, meta: Self::Metadata) -> Result { debug!("minimum_ledger_slot rpc request received"); meta.minimum_ledger_slot() @@ -4313,6 +4622,7 @@ pub mod tests { }, rpc_subscriptions::RpcSubscriptions, }, + base64::engine::general_purpose, bincode::deserialize, jsonrpc_core::{futures, ErrorCode, MetaIoHandler, Output, Response, Value}, jsonrpc_core_client::transports::local, @@ -5622,6 +5932,146 @@ pub mod tests { assert_eq!(result.len(), 0); } + #[test] + fn test_rpc_simulate_bundle_happy_path() { + // 1. setup + let rpc = RpcHandler::start(); + let bank = rpc.working_bank(); + + let recent_blockhash = bank.confirmed_last_blockhash(); + let RpcHandler { + ref meta, ref io, .. + } = rpc; + + let data_len = 100; + let lamports = bank.get_minimum_balance_for_rent_exemption(data_len); + let leader_pubkey = solana_sdk::pubkey::new_rand(); + let leader_account_data = AccountSharedData::new(lamports, data_len, &system_program::id()); + bank.store_account(&leader_pubkey, &leader_account_data); + bank.freeze(); + + // 2. build bundle + + // let's pretend the RPC keypair is a searcher + let searcher_keypair = rpc.mint_keypair; + + // create tip tx + let tip_amount = 10000; + let tip_tx = VersionedTransaction::from(system_transaction::transfer( + &searcher_keypair, + &leader_pubkey, + tip_amount, + recent_blockhash, + )); + + // some random mev tx + let mev_amount = 20000; + let goku_pubkey = solana_sdk::pubkey::new_rand(); + let mev_tx = VersionedTransaction::from(system_transaction::transfer( + &searcher_keypair, + &goku_pubkey, + mev_amount, + recent_blockhash, + )); + + let encoded_mev_tx = general_purpose::STANDARD.encode(serialize(&mev_tx).unwrap()); + let encoded_tip_tx = general_purpose::STANDARD.encode(serialize(&tip_tx).unwrap()); + let b64_data = general_purpose::STANDARD.encode(leader_account_data.data()); + + // 3. test and assert + let skip_sig_verify = true; + let replace_recent_blockhash = false; + let expected_response = json!({ + "jsonrpc": "2.0", + "result": { + "context": {"slot": bank.slot(), "apiVersion": RpcApiVersion::default()}, + "value":{ + "summary": "succeeded", + "transactionResults": [ + { + "err": null, + "logs": ["Program 11111111111111111111111111111111 invoke [1]", "Program 11111111111111111111111111111111 success"], + "returnData": null, + "unitsConsumed": 150, + "postExecutionAccounts": [], + "preExecutionAccounts": [ + { + "data": [b64_data, "base64"], + "executable": false, + "lamports": leader_account_data.lamports(), + "owner": "11111111111111111111111111111111", + "rentEpoch": 0, + "space": 100 + } + ], + }, + { + "err": null, + "logs": ["Program 11111111111111111111111111111111 invoke [1]", "Program 11111111111111111111111111111111 success"], + "returnData": null, + "unitsConsumed": 150, + "preExecutionAccounts": [], + "postExecutionAccounts": [ + { + "data": [b64_data, "base64"], + "executable": false, + "lamports": leader_account_data.lamports() + tip_amount, + "owner": "11111111111111111111111111111111", + "rentEpoch": u64::MAX, + "space": 100 + } + ], + }, + ], + } + }, + "id": 1, + }); + + let request = format!( + r#"{{"jsonrpc":"2.0", + "id":1, + "method":"simulateBundle", + "params":[ + {{ + "encodedTransactions": ["{}", "{}"] + }}, + {{ + "skipSigVerify": {}, + "replaceRecentBlockhash": {}, + "slot": {}, + "preExecutionAccountsConfigs": [ + {{ "encoding": "base64", "addresses": ["{}"] }}, + {{ "encoding": "base64", "addresses": [] }} + ], + "postExecutionAccountsConfigs": [ + {{ "encoding": "base64", "addresses": [] }}, + {{ "encoding": "base64", "addresses": ["{}"] }} + ] + }} + ] + }}"#, + encoded_mev_tx, + encoded_tip_tx, + skip_sig_verify, + replace_recent_blockhash, + bank.slot(), + leader_pubkey, + leader_pubkey, + ); + + let actual_response = io + .handle_request_sync(&request, meta.clone()) + .expect("response"); + + let expected_response = serde_json::from_value::(expected_response) + .expect("expected_response deserialization"); + let actual_response = serde_json::from_str::(&actual_response) + .expect("actual_response deserialization"); + + assert_eq!(expected_response, actual_response); + } + #[test] fn test_rpc_simulate_transaction() { let rpc = RpcHandler::start(); @@ -6440,10 +6890,7 @@ pub mod tests { ClusterInfo::new(contact_info, keypair, SocketAddrSpace::Unspecified) }); let connection_cache = Arc::new(ConnectionCache::new("connection_cache_test")); - let tpu_address = cluster_info - .my_contact_info() - .tpu(connection_cache.protocol()) - .unwrap(); + let (meta, receiver) = JsonRpcRequestProcessor::new( JsonRpcConfig::default(), None, @@ -6452,7 +6899,7 @@ pub mod tests { blockstore, validator_exit, health.clone(), - cluster_info, + cluster_info.clone(), Hash::default(), None, optimistically_confirmed_bank, @@ -6464,7 +6911,7 @@ pub mod tests { Arc::new(PrioritizationFeeCache::default()), ); SendTransactionService::new::( - tpu_address, + cluster_info, &bank_forks, None, receiver, @@ -6712,12 +7159,10 @@ pub mod tests { let cluster_info = Arc::new(new_test_cluster_info()); let connection_cache = Arc::new(ConnectionCache::new("connection_cache_test")); - let tpu_address = cluster_info - .my_contact_info() - .tpu(connection_cache.protocol()) - .unwrap(); + let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); + let (request_processor, receiver) = JsonRpcRequestProcessor::new( JsonRpcConfig::default(), None, @@ -6726,7 +7171,7 @@ pub mod tests { blockstore.clone(), validator_exit, RpcHealth::stub(optimistically_confirmed_bank.clone(), blockstore), - cluster_info, + cluster_info.clone(), Hash::default(), None, optimistically_confirmed_bank, @@ -6738,7 +7183,7 @@ pub mod tests { Arc::new(PrioritizationFeeCache::default()), ); SendTransactionService::new::( - tpu_address, + cluster_info, &bank_forks, None, receiver, diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index fed748d709..dc4353ddca 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -251,6 +251,7 @@ impl RequestMiddleware for RpcRequestMiddleware { let full_snapshot_archive_info = snapshot_utils::get_highest_full_snapshot_archive_info( &snapshot_config.full_snapshot_archives_dir, + None, ); let snapshot_archive_info = if let Some(full_snapshot_archive_info) = full_snapshot_archive_info { @@ -260,6 +261,7 @@ impl RequestMiddleware for RpcRequestMiddleware { snapshot_utils::get_highest_incremental_snapshot_archive_info( &snapshot_config.incremental_snapshot_archives_dir, full_snapshot_archive_info.slot(), + None, ) .map(|incremental_snapshot_archive_info| { incremental_snapshot_archive_info @@ -373,11 +375,6 @@ impl JsonRpcService { LARGEST_ACCOUNTS_CACHE_DURATION, ))); - let tpu_address = cluster_info - .my_contact_info() - .tpu(connection_cache.protocol()) - .map_err(|err| format!("{err}"))?; - // sadly, some parts of our current rpc implemention block the jsonrpc's // _socket-listening_ event loop for too long, due to (blocking) long IO or intesive CPU, // causing no further processing of incoming requests and ultimatily innocent clients timing-out. @@ -475,7 +472,7 @@ impl JsonRpcService { let leader_info = poh_recorder.map(|recorder| ClusterTpuInfo::new(cluster_info.clone(), recorder)); let _send_transaction_service = Arc::new(SendTransactionService::new_with_config( - tpu_address, + cluster_info, &bank_forks, leader_info, receiver, diff --git a/runtime-plugin/Cargo.toml b/runtime-plugin/Cargo.toml new file mode 100644 index 0000000000..2a526dc52e --- /dev/null +++ b/runtime-plugin/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "solana-runtime-plugin" +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +edition = { workspace = true } + +[dependencies] +crossbeam-channel = { workspace = true } +json5 = { workspace = true } +jsonrpc-core = { workspace = true } +jsonrpc-core-client = { workspace = true, features = ["ipc"] } +jsonrpc-derive = { workspace = true } +jsonrpc-ipc-server = { workspace = true } +jsonrpc-server-utils = { workspace = true } +libloading = { workspace = true } +log = { workspace = true } +solana-runtime = { workspace = true } +solana-sdk = { workspace = true } +thiserror = { workspace = true } diff --git a/runtime-plugin/src/lib.rs b/runtime-plugin/src/lib.rs new file mode 100644 index 0000000000..477af43c9b --- /dev/null +++ b/runtime-plugin/src/lib.rs @@ -0,0 +1,4 @@ +pub mod runtime_plugin; +pub mod runtime_plugin_admin_rpc_service; +pub mod runtime_plugin_manager; +pub mod runtime_plugin_service; diff --git a/runtime-plugin/src/runtime_plugin.rs b/runtime-plugin/src/runtime_plugin.rs new file mode 100644 index 0000000000..7dc0b95fa4 --- /dev/null +++ b/runtime-plugin/src/runtime_plugin.rs @@ -0,0 +1,41 @@ +use { + solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache}, + std::{ + any::Any, + error, + fmt::Debug, + io, + sync::{atomic::AtomicBool, Arc, RwLock}, + }, + thiserror::Error, +}; + +pub type Result = std::result::Result; + +/// Errors returned by plugin calls +#[derive(Error, Debug)] +pub enum RuntimePluginError { + /// Error opening the configuration file; for example, when the file + /// is not found or when the validator process has no permission to read it. + #[error("Error opening config file. Error detail: ({0}).")] + ConfigFileOpenError(#[from] io::Error), + + /// Any custom error defined by the plugin. + #[error("Plugin-defined custom error. Error message: ({0})")] + Custom(Box), + + #[error("Failed to load a runtime plugin")] + FailedToLoadPlugin(#[from] Box), +} + +pub struct PluginDependencies { + pub bank_forks: Arc>, + pub block_commitment_cache: Arc>, + pub exit: Arc, +} + +pub trait RuntimePlugin: Any + Debug + Send + Sync { + fn name(&self) -> &'static str; + fn on_load(&mut self, config_file: &str, dependencies: PluginDependencies) -> Result<()>; + fn on_unload(&mut self); +} diff --git a/runtime-plugin/src/runtime_plugin_admin_rpc_service.rs b/runtime-plugin/src/runtime_plugin_admin_rpc_service.rs new file mode 100644 index 0000000000..fdc33b06c5 --- /dev/null +++ b/runtime-plugin/src/runtime_plugin_admin_rpc_service.rs @@ -0,0 +1,326 @@ +//! RPC interface to dynamically make changes to runtime plugins. + +use { + crossbeam_channel::Sender, + jsonrpc_core::{BoxFuture, ErrorCode, MetaIoHandler, Metadata, Result as JsonRpcResult}, + jsonrpc_core_client::{transports::ipc, RpcError}, + jsonrpc_derive::rpc, + jsonrpc_ipc_server::{ + tokio::{self, sync::oneshot::channel as oneshot_channel}, + RequestContext, ServerBuilder, + }, + jsonrpc_server_utils::tokio::sync::oneshot::Sender as OneShotSender, + log::*, + solana_sdk::exit::Exit, + std::{ + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, RwLock, + }, + }, +}; + +#[derive(Debug)] +pub enum RuntimePluginManagerRpcRequest { + ReloadPlugin { + name: String, + config_file: String, + response_sender: OneShotSender>, + }, + UnloadPlugin { + name: String, + response_sender: OneShotSender>, + }, + LoadPlugin { + config_file: String, + response_sender: OneShotSender>, + }, + ListPlugins { + response_sender: OneShotSender>>, + }, +} + +#[rpc] +pub trait RuntimePluginAdminRpc { + type Metadata; + + #[rpc(meta, name = "reloadPlugin")] + fn reload_plugin( + &self, + meta: Self::Metadata, + name: String, + config_file: String, + ) -> BoxFuture>; + + #[rpc(meta, name = "unloadPlugin")] + fn unload_plugin(&self, meta: Self::Metadata, name: String) -> BoxFuture>; + + #[rpc(meta, name = "loadPlugin")] + fn load_plugin( + &self, + meta: Self::Metadata, + config_file: String, + ) -> BoxFuture>; + + #[rpc(meta, name = "listPlugins")] + fn list_plugins(&self, meta: Self::Metadata) -> BoxFuture>>; +} + +#[derive(Clone)] +pub struct RuntimePluginAdminRpcRequestMetadata { + pub rpc_request_sender: Sender, + pub validator_exit: Arc>, +} + +impl Metadata for RuntimePluginAdminRpcRequestMetadata {} + +fn rpc_path(ledger_path: &Path) -> PathBuf { + #[cfg(target_family = "windows")] + { + // More information about the wackiness of pipe names over at + // https://docs.microsoft.com/en-us/windows/win32/ipc/pipe-names + if let Some(ledger_filename) = ledger_path.file_name() { + PathBuf::from(format!( + "\\\\.\\pipe\\{}-runtime_plugin_admin.rpc", + ledger_filename.to_string_lossy() + )) + } else { + PathBuf::from("\\\\.\\pipe\\runtime_plugin_admin.rpc") + } + } + #[cfg(not(target_family = "windows"))] + { + ledger_path.join("runtime_plugin_admin.rpc") + } +} + +/// Start the Runtime Plugin Admin RPC interface. +pub fn run( + ledger_path: &Path, + metadata: RuntimePluginAdminRpcRequestMetadata, + plugin_exit: Arc, +) { + let rpc_path = rpc_path(ledger_path); + + let event_loop = tokio::runtime::Builder::new_multi_thread() + .thread_name("solRuntimePluginAdminRpc") + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + + std::thread::Builder::new() + .name("solAdminRpc".to_string()) + .spawn(move || { + let mut io = MetaIoHandler::default(); + io.extend_with(RuntimePluginAdminRpcImpl.to_delegate()); + + let validator_exit = metadata.validator_exit.clone(); + + match ServerBuilder::with_meta_extractor(io, move |_req: &RequestContext| { + metadata.clone() + }) + .event_loop_executor(event_loop.handle().clone()) + .start(&format!("{}", rpc_path.display())) + { + Err(e) => { + error!("Unable to start runtime plugin admin rpc service: {e:?}, exiting"); + validator_exit.write().unwrap().exit(); + } + Ok(server) => { + info!("started runtime plugin admin rpc service!"); + let close_handle = server.close_handle(); + let c_plugin_exit = plugin_exit.clone(); + validator_exit + .write() + .unwrap() + .register_exit(Box::new(move || { + close_handle.close(); + c_plugin_exit.store(true, Ordering::Relaxed); + })); + + server.wait(); + plugin_exit.store(true, Ordering::Relaxed); + } + } + }) + .unwrap(); +} + +pub struct RuntimePluginAdminRpcImpl; +impl RuntimePluginAdminRpc for RuntimePluginAdminRpcImpl { + type Metadata = RuntimePluginAdminRpcRequestMetadata; + + fn reload_plugin( + &self, + meta: Self::Metadata, + name: String, + config_file: String, + ) -> BoxFuture> { + Box::pin(async move { + let (response_sender, response_receiver) = oneshot_channel(); + + if meta + .rpc_request_sender + .send(RuntimePluginManagerRpcRequest::ReloadPlugin { + name, + config_file, + response_sender, + }) + .is_err() + { + error!("rpc_request_sender channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + + return Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while sending the request".to_string(), + data: None, + }); + } + + match response_receiver.await { + Err(_) => { + error!("response_receiver channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while awaiting the response" + .to_string(), + data: None, + }) + } + Ok(resp) => resp, + } + }) + } + + fn unload_plugin(&self, meta: Self::Metadata, name: String) -> BoxFuture> { + Box::pin(async move { + let (response_sender, response_receiver) = oneshot_channel(); + + if meta + .rpc_request_sender + .send(RuntimePluginManagerRpcRequest::UnloadPlugin { + name, + response_sender, + }) + .is_err() + { + error!("rpc_request_sender channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + + return Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while sending the request".to_string(), + data: None, + }); + } + + match response_receiver.await { + Err(_) => { + error!("response_receiver channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while awaiting the response" + .to_string(), + data: None, + }) + } + Ok(resp) => resp, + } + }) + } + + fn load_plugin( + &self, + meta: Self::Metadata, + config_file: String, + ) -> BoxFuture> { + Box::pin(async move { + let (response_sender, response_receiver) = oneshot_channel(); + + if meta + .rpc_request_sender + .send(RuntimePluginManagerRpcRequest::LoadPlugin { + config_file, + response_sender, + }) + .is_err() + { + error!("rpc_request_sender channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + + return Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while sending the request".to_string(), + data: None, + }); + } + + match response_receiver.await { + Err(_) => { + error!("response_receiver channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while awaiting the response" + .to_string(), + data: None, + }) + } + Ok(resp) => resp, + } + }) + } + + fn list_plugins(&self, meta: Self::Metadata) -> BoxFuture>> { + Box::pin(async move { + let (response_sender, response_receiver) = oneshot_channel(); + + if meta + .rpc_request_sender + .send(RuntimePluginManagerRpcRequest::ListPlugins { response_sender }) + .is_err() + { + error!("rpc_request_sender channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + + return Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while sending the request".to_string(), + data: None, + }); + } + + match response_receiver.await { + Err(_) => { + error!("response_receiver channel closed, exiting"); + meta.validator_exit.write().unwrap().exit(); + Err(jsonrpc_core::Error { + code: ErrorCode::InternalError, + message: "Internal channel disconnected while awaiting the response" + .to_string(), + data: None, + }) + } + Ok(resp) => resp, + } + }) + } +} + +// Connect to the Runtime Plugin RPC interface +pub async fn connect(ledger_path: &Path) -> Result { + let rpc_path = rpc_path(ledger_path); + if !rpc_path.exists() { + Err(RpcError::Client(format!( + "{} does not exist", + rpc_path.display() + ))) + } else { + ipc::connect::<_, gen_client::Client>(&format!("{}", rpc_path.display())).await + } +} diff --git a/runtime-plugin/src/runtime_plugin_manager.rs b/runtime-plugin/src/runtime_plugin_manager.rs new file mode 100644 index 0000000000..af1dcf2cde --- /dev/null +++ b/runtime-plugin/src/runtime_plugin_manager.rs @@ -0,0 +1,275 @@ +use { + crate::runtime_plugin::{PluginDependencies, RuntimePlugin}, + jsonrpc_core::{serde_json, ErrorCode, Result as JsonRpcResult}, + libloading::Library, + log::*, + solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache}, + std::{ + fs::File, + io::Read, + path::{Path, PathBuf}, + sync::{atomic::AtomicBool, Arc, RwLock}, + }, +}; + +#[derive(thiserror::Error, Debug)] +pub enum RuntimePluginManagerError { + #[error("Cannot open the the plugin config file")] + CannotOpenConfigFile(String), + + #[error("Cannot read the the plugin config file")] + CannotReadConfigFile(String), + + #[error("The config file is not in a valid Json format")] + InvalidConfigFileFormat(String), + + #[error("Plugin library path is not specified in the config file")] + LibPathNotSet, + + #[error("Invalid plugin path")] + InvalidPluginPath, + + #[error("Cannot load plugin shared library")] + PluginLoadError(String), + + #[error("The runtime plugin {0} is already loaded shared library")] + PluginAlreadyLoaded(String), + + #[error("The RuntimePlugin on_load method failed")] + PluginStartError(String), +} + +pub struct RuntimePluginManager { + plugins: Vec>, + libs: Vec, + bank_forks: Arc>, + block_commitment_cache: Arc>, + exit: Arc, +} + +impl RuntimePluginManager { + pub fn new( + bank_forks: Arc>, + block_commitment_cache: Arc>, + exit: Arc, + ) -> Self { + Self { + plugins: vec![], + libs: vec![], + bank_forks, + block_commitment_cache, + exit, + } + } + + /// This method allows dynamic loading of a runtime plugin. + /// Adds to the existing list of loaded plugins. + pub(crate) fn load_plugin( + &mut self, + plugin_config_path: impl AsRef, + ) -> JsonRpcResult { + // First load plugin + let (mut new_plugin, new_lib, config_file) = + load_plugin_from_config(plugin_config_path.as_ref()).map_err(|e| { + jsonrpc_core::Error { + code: ErrorCode::InvalidRequest, + message: format!("Failed to load plugin: {e}"), + data: None, + } + })?; + + // Then see if a plugin with this name already exists, if so return Err. + let name = new_plugin.name(); + if self.plugins.iter().any(|plugin| name.eq(plugin.name())) { + return Err(jsonrpc_core::Error { + code: ErrorCode::InvalidRequest, + message: format!( + "There already exists a plugin named {} loaded. Did not load requested plugin", + name, + ), + data: None, + }); + } + + new_plugin + .on_load( + config_file, + PluginDependencies { + bank_forks: self.bank_forks.clone(), + block_commitment_cache: self.block_commitment_cache.clone(), + exit: self.exit.clone(), + }, + ) + .map_err(|on_load_err| jsonrpc_core::Error { + code: ErrorCode::InvalidRequest, + message: format!( + "on_load method of plugin {} failed: {on_load_err}", + new_plugin.name() + ), + data: None, + })?; + + self.plugins.push(new_plugin); + self.libs.push(new_lib); + + Ok(name.to_string()) + } + + /// Unloads the plugins and loaded plugin libraries, making sure to fire + /// their `on_plugin_unload()` methods so they can do any necessary cleanup. + pub(crate) fn unload_all_plugins(&mut self) { + (0..self.plugins.len()).for_each(|idx| { + self.try_drop_plugin(idx); + }); + } + + pub(crate) fn unload_plugin(&mut self, name: &str) -> JsonRpcResult<()> { + // Check if any plugin names match this one + let Some(idx) = self + .plugins + .iter() + .position(|plugin| plugin.name().eq(name)) + else { + // If we don't find one return an error + return Err(jsonrpc_core::error::Error { + code: ErrorCode::InvalidRequest, + message: String::from("The plugin you requested to unload is not loaded"), + data: None, + }); + }; + + // Unload and drop plugin and lib + self.try_drop_plugin(idx); + + Ok(()) + } + + /// Reloads an existing plugin. + pub(crate) fn reload_plugin(&mut self, name: &str, config_file: &str) -> JsonRpcResult<()> { + // Check if any plugin names match this one + let Some(idx) = self + .plugins + .iter() + .position(|plugin| plugin.name().eq(name)) + else { + // If we don't find one return an error + return Err(jsonrpc_core::error::Error { + code: ErrorCode::InvalidRequest, + message: String::from("The plugin you requested to reload is not loaded"), + data: None, + }); + }; + + self.try_drop_plugin(idx); + + // Try to load plugin, library + // SAFETY: It is up to the validator to ensure this is a valid plugin library. + let (mut new_plugin, new_lib, new_parsed_config_file) = + load_plugin_from_config(config_file.as_ref()).map_err(|err| jsonrpc_core::Error { + code: ErrorCode::InvalidRequest, + message: err.to_string(), + data: None, + })?; + + // Attempt to on_load with new plugin + match new_plugin.on_load( + new_parsed_config_file, + PluginDependencies { + bank_forks: self.bank_forks.clone(), + block_commitment_cache: self.block_commitment_cache.clone(), + exit: self.exit.clone(), + }, + ) { + // On success, push plugin and library + Ok(()) => { + self.plugins.push(new_plugin); + self.libs.push(new_lib); + Ok(()) + } + // On failure, return error + Err(err) => Err(jsonrpc_core::error::Error { + code: ErrorCode::InvalidRequest, + message: format!( + "Failed to start new plugin (previous plugin was dropped!): {err}" + ), + data: None, + }), + } + } + + pub(crate) fn list_plugins(&self) -> JsonRpcResult> { + Ok(self.plugins.iter().map(|p| p.name().to_owned()).collect()) + } + + fn try_drop_plugin(&mut self, idx: usize) { + if idx < self.plugins.len() { + let mut plugin = self.plugins.remove(idx); + let lib = self.libs.remove(idx); + drop(lib); + plugin.on_unload(); + } else { + error!("failed to drop plugin: index {idx} out of bounds"); + } + } +} + +fn load_plugin_from_config( + plugin_config_path: &Path, +) -> Result<(Box, Library, &str), RuntimePluginManagerError> { + type PluginConstructor = unsafe fn() -> *mut dyn RuntimePlugin; + use libloading::Symbol; + + let mut file = match File::open(plugin_config_path) { + Ok(file) => file, + Err(err) => { + return Err(RuntimePluginManagerError::CannotOpenConfigFile(format!( + "Failed to open the plugin config file {plugin_config_path:?}, error: {err:?}" + ))); + } + }; + + let mut contents = String::new(); + if let Err(err) = file.read_to_string(&mut contents) { + return Err(RuntimePluginManagerError::CannotReadConfigFile(format!( + "Failed to read the plugin config file {plugin_config_path:?}, error: {err:?}" + ))); + } + + let result: serde_json::Value = match json5::from_str(&contents) { + Ok(value) => value, + Err(err) => { + return Err(RuntimePluginManagerError::InvalidConfigFileFormat(format!( + "The config file {plugin_config_path:?} is not in a valid Json5 format, error: {err:?}" + ))); + } + }; + + let libpath = result["libpath"] + .as_str() + .ok_or(RuntimePluginManagerError::LibPathNotSet)?; + let mut libpath = PathBuf::from(libpath); + if libpath.is_relative() { + let config_dir = plugin_config_path.parent().ok_or_else(|| { + RuntimePluginManagerError::CannotOpenConfigFile(format!( + "Failed to resolve parent of {plugin_config_path:?}", + )) + })?; + libpath = config_dir.join(libpath); + } + + let config_file = plugin_config_path + .as_os_str() + .to_str() + .ok_or(RuntimePluginManagerError::InvalidPluginPath)?; + + let (plugin, lib) = unsafe { + let lib = Library::new(libpath) + .map_err(|e| RuntimePluginManagerError::PluginLoadError(e.to_string()))?; + let constructor: Symbol = lib + .get(b"_create_plugin") + .map_err(|e| RuntimePluginManagerError::PluginLoadError(e.to_string()))?; + (Box::from_raw(constructor()), lib) + }; + + Ok((plugin, lib, config_file)) +} diff --git a/runtime-plugin/src/runtime_plugin_service.rs b/runtime-plugin/src/runtime_plugin_service.rs new file mode 100644 index 0000000000..5fcb625a26 --- /dev/null +++ b/runtime-plugin/src/runtime_plugin_service.rs @@ -0,0 +1,123 @@ +use { + crate::{ + runtime_plugin::RuntimePluginError, + runtime_plugin_admin_rpc_service::RuntimePluginManagerRpcRequest, + runtime_plugin_manager::RuntimePluginManager, + }, + crossbeam_channel::Receiver, + log::{error, info}, + solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache}, + std::{ + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, RwLock, + }, + thread::{self, JoinHandle}, + time::Duration, + }, +}; + +pub struct RuntimePluginService { + plugin_manager: Arc>, + rpc_thread: JoinHandle<()>, +} + +impl RuntimePluginService { + pub fn start( + plugin_config_files: &[PathBuf], + rpc_receiver: Receiver, + bank_forks: Arc>, + block_commitment_cache: Arc>, + exit: Arc, + ) -> Result { + let mut plugin_manager = + RuntimePluginManager::new(bank_forks, block_commitment_cache, exit.clone()); + + for config in plugin_config_files { + let name = plugin_manager + .load_plugin(config) + .map_err(|e| RuntimePluginError::FailedToLoadPlugin(e.into()))?; + info!("Loaded Runtime Plugin: {name}"); + } + + let plugin_manager = Arc::new(RwLock::new(plugin_manager)); + let rpc_thread = + Self::start_rpc_request_handler(rpc_receiver, plugin_manager.clone(), exit); + + Ok(Self { + plugin_manager, + rpc_thread, + }) + } + + pub fn join(self) { + if let Err(e) = self.rpc_thread.join() { + error!("error joining rpc thread: {e:?}"); + } + self.plugin_manager.write().unwrap().unload_all_plugins(); + } + + fn start_rpc_request_handler( + rpc_receiver: Receiver, + plugin_manager: Arc>, + exit: Arc, + ) -> JoinHandle<()> { + thread::Builder::new() + .name("solRuntimePluginRpc".to_string()) + .spawn(move || { + const TIMEOUT: Duration = Duration::from_secs(3); + while !exit.load(Ordering::Relaxed) { + if let Ok(request) = rpc_receiver.recv_timeout(TIMEOUT) { + match request { + RuntimePluginManagerRpcRequest::ListPlugins { response_sender } => { + let plugin_list = plugin_manager.read().unwrap().list_plugins(); + if response_sender.send(plugin_list).is_err() { + error!("response_sender channel disconnected"); + return; + } + } + RuntimePluginManagerRpcRequest::ReloadPlugin { + ref name, + ref config_file, + response_sender, + } => { + let reload_result = plugin_manager + .write() + .unwrap() + .reload_plugin(name, config_file); + if response_sender.send(reload_result).is_err() { + error!("response_sender channel disconnected"); + return; + } + } + RuntimePluginManagerRpcRequest::LoadPlugin { + ref config_file, + response_sender, + } => { + let load_result = + plugin_manager.write().unwrap().load_plugin(config_file); + if response_sender.send(load_result).is_err() { + error!("response_sender channel disconnected"); + return; + } + } + RuntimePluginManagerRpcRequest::UnloadPlugin { + ref name, + response_sender, + } => { + let unload_result = + plugin_manager.write().unwrap().unload_plugin(name); + if response_sender.send(unload_result).is_err() { + error!("response_sender channel disconnected"); + return; + } + } + } + } + } + plugin_manager.write().unwrap().unload_all_plugins(); + }) + .unwrap() + } +} diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index c5a629d030..71c73e16fc 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -67,7 +67,7 @@ use { }, serde::Serialize, solana_accounts_db::{ - accounts::{AccountAddressFilter, Accounts, PubkeyAccountSlot}, + accounts::{AccountAddressFilter, AccountLocks, Accounts, PubkeyAccountSlot}, accounts_db::{ AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, CalcAccountsHashDataSource, PubkeyHashAccount, VerifyAccountsHashAndLamportsConfig, @@ -323,6 +323,7 @@ impl BankRc { } } +#[derive(Debug)] pub struct LoadAndExecuteTransactionsOutput { pub loaded_transactions: Vec, // Vector of results indicating whether a transaction was executed or could not @@ -340,6 +341,22 @@ pub struct LoadAndExecuteTransactionsOutput { pub error_counters: TransactionErrorMetrics, } +#[derive(Clone)] +pub struct BundleTransactionSimulationResult { + pub result: Result<()>, + pub logs: TransactionLogMessages, + pub pre_execution_accounts: Option>, + pub post_execution_accounts: Option>, + pub return_data: Option, + pub units_consumed: u64, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AccountData { + pub pubkey: Pubkey, + pub data: AccountSharedData, +} + pub struct TransactionSimulationResult { pub result: Result<()>, pub logs: TransactionLogMessages, @@ -781,7 +798,7 @@ pub struct Bank { inflation: Arc>, /// cache of vote_account and stake_account state for this fork - stakes_cache: StakesCache, + pub stakes_cache: StakesCache, /// staked nodes on epoch boundaries, saved off when a bank.slot() is at /// a leader schedule calculation boundary @@ -3278,17 +3295,61 @@ impl Bank { &'a self, transactions: &'b [SanitizedTransaction], transaction_results: impl Iterator>, + additional_read_locks: Option<&HashSet>, + additional_write_locks: Option<&HashSet>, ) -> TransactionBatch<'a, 'b> { - // this lock_results could be: Ok, AccountInUse, WouldExceedBlockMaxLimit or WouldExceedAccountMaxLimit let tx_account_lock_limit = self.get_transaction_account_lock_limit(); let lock_results = self.rc.accounts.lock_accounts_with_results( transactions.iter(), transaction_results, tx_account_lock_limit, + additional_read_locks, + additional_write_locks, ); TransactionBatch::new(lock_results, self, Cow::Borrowed(transactions)) } + /// Prepare a locked transaction batch from a list of sanitized transactions, and their cost + /// limited packing status, where transactions will be locked sequentially until the first failure + pub fn prepare_sequential_sanitized_batch_with_results<'a, 'b>( + &'a self, + transactions: &'b [SanitizedTransaction], + ) -> TransactionBatch<'a, 'b> { + // this lock_results could be: Ok, AccountInUse, AccountLoadedTwice, or TooManyAccountLocks + let tx_account_lock_limit = self.get_transaction_account_lock_limit(); + let lock_results = self + .rc + .accounts + .lock_accounts_sequential_with_results(transactions.iter(), tx_account_lock_limit); + TransactionBatch::new(lock_results, self, Cow::Borrowed(transactions)) + } + + /// Prepare a locked transaction batch from a list of sanitized transactions for simulation. + /// This grabs as many sequential account locks that it can without a RW conflict. However, + /// it uses a temporary version of AccountLocks and not the Bank's account locks, so one can + /// use this during simulation on an unfrozen Bank without worrying about impacting the RW + /// lock usage in replay + pub fn prepare_sequential_sanitized_batch_with_results_for_simulation<'a, 'b>( + &'a self, + transactions: &'b [SanitizedTransaction], + ) -> TransactionBatch<'a, 'b> { + let tx_account_lock_limit = self.get_transaction_account_lock_limit(); + let tx_account_locks_results: Vec> = transactions + .iter() + .map(|tx| tx.get_account_locks(tx_account_lock_limit)) + .collect(); + + let mut account_locks = AccountLocks::default(); + let lock_results = + Accounts::lock_accounts_sequential(&mut account_locks, tx_account_locks_results); + let mut batch = TransactionBatch::new(lock_results, self, Cow::Borrowed(transactions)); + // this is required to ensure that accounts aren't unlocked accidentally, which can be problematic during replay. + // more specifically, during process_entries, if the lock counts are accidentally decremented, + // one might end up replaying a block incorrectly + batch.set_needs_unlock(false); + batch + } + /// Prepare a transaction batch from a single transaction without locking accounts pub fn prepare_unlocked_batch_from_single_tx<'a>( &'a self, @@ -3410,7 +3471,11 @@ impl Bank { } } - fn get_account_overrides_for_simulation(&self, account_keys: &AccountKeys) -> AccountOverrides { + // NOTE: Do not revert this back to private during rebases. + pub fn get_account_overrides_for_simulation( + &self, + account_keys: &AccountKeys, + ) -> AccountOverrides { let mut account_overrides = AccountOverrides::default(); let slot_history_id = sysvar::slot_history::id(); if account_keys.iter().any(|pubkey| *pubkey == slot_history_id) { @@ -3598,6 +3663,30 @@ impl Bank { balances } + pub fn collect_balances_with_cache( + &self, + batch: &TransactionBatch, + account_overrides: Option<&AccountOverrides>, + ) -> TransactionBalances { + let mut balances: TransactionBalances = vec![]; + for transaction in batch.sanitized_transactions() { + let mut transaction_balances: Vec = vec![]; + for account_key in transaction.message().account_keys().iter() { + let balance = match account_overrides { + None => self.get_balance(account_key), + Some(overrides) => match overrides.get(account_key) { + None => self.get_balance(account_key), + Some(account_data) => account_data.lamports(), + }, + }; + transaction_balances.push(balance); + } + balances.push(transaction_balances); + } + balances + } + + #[allow(clippy::too_many_arguments, clippy::type_complexity)] pub fn load_and_execute_transactions( &self, batch: &TransactionBatch, diff --git a/runtime/src/snapshot_bank_utils.rs b/runtime/src/snapshot_bank_utils.rs index 5b3f83f28a..8e42d859a7 100644 --- a/runtime/src/snapshot_bank_utils.rs +++ b/runtime/src/snapshot_bank_utils.rs @@ -87,13 +87,14 @@ pub fn bank_fields_from_snapshot_archives( storage_access: StorageAccess, ) -> snapshot_utils::Result { let full_snapshot_archive_info = - get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir).ok_or_else(|| { - SnapshotError::NoSnapshotArchives(full_snapshot_archives_dir.as_ref().to_path_buf()) - })?; + get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir, None).ok_or_else( + || SnapshotError::NoSnapshotArchives(full_snapshot_archives_dir.as_ref().to_path_buf()), + )?; let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info( &incremental_snapshot_archives_dir, full_snapshot_archive_info.slot(), + None, ); let temp_unpack_dir = TempDir::new()?; @@ -298,13 +299,14 @@ pub fn bank_from_latest_snapshot_archives( Option, )> { let full_snapshot_archive_info = - get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir).ok_or_else(|| { - SnapshotError::NoSnapshotArchives(full_snapshot_archives_dir.as_ref().to_path_buf()) - })?; + get_highest_full_snapshot_archive_info(&full_snapshot_archives_dir, None).ok_or_else( + || SnapshotError::NoSnapshotArchives(full_snapshot_archives_dir.as_ref().to_path_buf()), + )?; let incremental_snapshot_archive_info = get_highest_incremental_snapshot_archive_info( &incremental_snapshot_archives_dir, full_snapshot_archive_info.slot(), + None, ); let (bank, _) = bank_from_snapshot_archives( @@ -2550,7 +2552,7 @@ mod tests { fs::remove_file(full_snapshot_archive_info.unwrap().path()).unwrap(); let highest_full_snapshot_archive = - get_highest_full_snapshot_archive_info(&snapshot_archives_dir).unwrap(); + get_highest_full_snapshot_archive_info(&snapshot_archives_dir, None).unwrap(); let highest_bank_snapshot_post = get_highest_bank_snapshot_post(&bank_snapshots_dir).unwrap(); let highest_bank_snapshot_pre = get_highest_bank_snapshot_pre(&bank_snapshots_dir).unwrap(); diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index 5d407eaf0b..7e0b55d9fd 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -701,7 +701,7 @@ pub fn get_highest_loadable_bank_snapshot( // Otherwise, the bank snapshot's full snapshot slot *must* be the same as // the highest full snapshot archive's slot. let highest_full_snapshot_archive_slot = - get_highest_full_snapshot_archive_slot(&snapshot_config.full_snapshot_archives_dir)?; + get_highest_full_snapshot_archive_slot(&snapshot_config.full_snapshot_archives_dir, None)?; let full_snapshot_file_slot = read_full_snapshot_slot_file(&highest_bank_snapshot.snapshot_dir).ok()?; (full_snapshot_file_slot == highest_full_snapshot_archive_slot).then_some(highest_bank_snapshot) @@ -2066,8 +2066,9 @@ pub fn get_incremental_snapshot_archives( /// Get the highest slot of the full snapshot archives in a directory pub fn get_highest_full_snapshot_archive_slot( full_snapshot_archives_dir: impl AsRef, + halt_at_slot: Option, ) -> Option { - get_highest_full_snapshot_archive_info(full_snapshot_archives_dir) + get_highest_full_snapshot_archive_info(full_snapshot_archives_dir, halt_at_slot) .map(|full_snapshot_archive_info| full_snapshot_archive_info.slot()) } @@ -2076,10 +2077,12 @@ pub fn get_highest_full_snapshot_archive_slot( pub fn get_highest_incremental_snapshot_archive_slot( incremental_snapshot_archives_dir: impl AsRef, full_snapshot_slot: Slot, + halt_at_slot: Option, ) -> Option { get_highest_incremental_snapshot_archive_info( incremental_snapshot_archives_dir, full_snapshot_slot, + halt_at_slot, ) .map(|incremental_snapshot_archive_info| incremental_snapshot_archive_info.slot()) } @@ -2087,8 +2090,13 @@ pub fn get_highest_incremental_snapshot_archive_slot( /// Get the path (and metadata) for the full snapshot archive with the highest slot in a directory pub fn get_highest_full_snapshot_archive_info( full_snapshot_archives_dir: impl AsRef, + halt_at_slot: Option, ) -> Option { let mut full_snapshot_archives = get_full_snapshot_archives(full_snapshot_archives_dir); + if let Some(halt_at_slot) = halt_at_slot { + full_snapshot_archives + .retain(|archive| archive.snapshot_archive_info().slot <= halt_at_slot); + } full_snapshot_archives.sort_unstable(); full_snapshot_archives.into_iter().next_back() } @@ -2098,6 +2106,7 @@ pub fn get_highest_full_snapshot_archive_info( pub fn get_highest_incremental_snapshot_archive_info( incremental_snapshot_archives_dir: impl AsRef, full_snapshot_slot: Slot, + halt_at_slot: Option, ) -> Option { // Since we want to filter down to only the incremental snapshot archives that have the same // full snapshot slot as the value passed in, perform the filtering before sorting to avoid @@ -2109,6 +2118,9 @@ pub fn get_highest_incremental_snapshot_archive_info( incremental_snapshot_archive_info.base_slot() == full_snapshot_slot }) .collect::>(); + if let Some(halt_at_slot) = halt_at_slot { + incremental_snapshot_archives.retain(|archive| archive.slot() <= halt_at_slot); + } incremental_snapshot_archives.sort_unstable(); incremental_snapshot_archives.into_iter().next_back() } @@ -3136,7 +3148,7 @@ mod tests { ); assert_eq!( - get_highest_full_snapshot_archive_slot(full_snapshot_archives_dir.path()), + get_highest_full_snapshot_archive_slot(full_snapshot_archives_dir.path(), None), Some(max_slot - 1) ); } @@ -3162,7 +3174,8 @@ mod tests { assert_eq!( get_highest_incremental_snapshot_archive_slot( incremental_snapshot_archives_dir.path(), - full_snapshot_slot + full_snapshot_slot, + None, ), Some(max_incremental_snapshot_slot - 1) ); @@ -3171,7 +3184,8 @@ mod tests { assert_eq!( get_highest_incremental_snapshot_archive_slot( incremental_snapshot_archives_dir.path(), - max_full_snapshot_slot + max_full_snapshot_slot, + None, ), None ); diff --git a/runtime/src/stake_account.rs b/runtime/src/stake_account.rs index ea4ed6dd0f..017b6fb52f 100644 --- a/runtime/src/stake_account.rs +++ b/runtime/src/stake_account.rs @@ -41,14 +41,14 @@ impl StakeAccount { } #[inline] - pub(crate) fn stake_state(&self) -> &StakeStateV2 { + pub fn stake_state(&self) -> &StakeStateV2 { &self.stake_state } } impl StakeAccount { #[inline] - pub(crate) fn delegation(&self) -> Delegation { + pub fn delegation(&self) -> Delegation { // Safe to unwrap here because StakeAccount will always // only wrap a stake-state which is a delegation. self.stake_state.delegation().unwrap() diff --git a/runtime/src/stakes.rs b/runtime/src/stakes.rs index 27ec9d683c..08a7ed9581 100644 --- a/runtime/src/stakes.rs +++ b/runtime/src/stakes.rs @@ -49,18 +49,18 @@ pub enum InvalidCacheEntryReason { WrongOwner, } -type StakeAccount = stake_account::StakeAccount; +pub type StakeAccount = stake_account::StakeAccount; #[cfg_attr(feature = "frozen-abi", derive(AbiExample))] #[derive(Default, Debug)] -pub(crate) struct StakesCache(RwLock>); +pub struct StakesCache(RwLock>); impl StakesCache { pub(crate) fn new(stakes: Stakes) -> Self { Self(RwLock::new(stakes)) } - pub(crate) fn stakes(&self) -> RwLockReadGuard> { + pub fn stakes(&self) -> RwLockReadGuard> { self.0.read().unwrap() } @@ -188,7 +188,7 @@ pub struct Stakes { vote_accounts: VoteAccounts, /// stake_delegations - stake_delegations: ImHashMap, + pub stake_delegations: ImHashMap, /// unused unused: u64, @@ -231,7 +231,7 @@ impl Stakes { /// full account state for respective stake pubkeys. get_account function /// should return the account at the respective slot where stakes where /// cached. - pub(crate) fn new(stakes: &Stakes, get_account: F) -> Result + pub fn new(stakes: &Stakes, get_account: F) -> Result where F: Fn(&Pubkey) -> Option + Sync, { @@ -471,7 +471,7 @@ impl Stakes { ); } - pub(crate) fn stake_delegations(&self) -> &ImHashMap { + pub fn stake_delegations(&self) -> &ImHashMap { &self.stake_delegations } diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index ecec27e02e..3e92176db2 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -1,6 +1,6 @@ use { crate::bank::Bank, - solana_sdk::transaction::{Result, SanitizedTransaction}, + solana_sdk::transaction::{Result, SanitizedTransaction, TransactionError}, std::borrow::Cow, }; @@ -79,6 +79,28 @@ impl<'a, 'b> TransactionBatch<'a, 'b> { // that validity constraint. self.lock_results = transaction_results; } + + /// Bundle locking failed if lock result returns something other than ok or AccountInUse + pub fn check_bundle_lock_results(&self) -> Option<(&SanitizedTransaction, &TransactionError)> { + self.sanitized_transactions() + .iter() + .zip(self.lock_results.iter()) + .find(|(_, lock_result)| { + !matches!(lock_result, Ok(()) | Err(TransactionError::AccountInUse)) + }) + .map(|(transaction, lock_result)| { + ( + transaction, + match lock_result { + Ok(_) => { + // safe here bc the above find will never return Ok + unreachable!() + } + Err(lock_error) => lock_error, + }, + ) + }) + } } // Unlock all locked accounts in destructor. diff --git a/rustfmt.toml b/rustfmt.toml index e26d07f0d8..c7ccd48750 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,2 +1,7 @@ imports_granularity = "One" group_imports = "One" + +ignore = [ + "jito-programs", + "anchor" +] \ No newline at end of file diff --git a/s b/s new file mode 100755 index 0000000000..308133d227 --- /dev/null +++ b/s @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +if [ -f .env ]; then + export $(cat .env | grep -v '#' | awk '/=/ {print $1}') +else + echo "Missing .env file" + exit 0 +fi + +echo "Syncing to host: $HOST" + +# sync to build server, ignoring local builds and local/remote dev ledger +rsync -avh --delete --exclude target --exclude docker-output "$SCRIPT_DIR" "$HOST":~/ diff --git a/scripts/agave-install-deploy.sh b/scripts/agave-install-deploy.sh index dcdec14ffb..86575954d3 100755 --- a/scripts/agave-install-deploy.sh +++ b/scripts/agave-install-deploy.sh @@ -57,10 +57,10 @@ esac case $TAG in edge|beta) - DOWNLOAD_URL=https://release.anza.xyz/"$TAG"/solana-release-$TARGET.tar.bz2 + DOWNLOAD_URL=https://release.jito.wtf/"$TAG"/solana-release-$TARGET.tar.bz2 ;; *) - DOWNLOAD_URL=https://github.com/anza-xyz/agave/releases/download/"$TAG"/solana-release-$TARGET.tar.bz2 + DOWNLOAD_URL=https://github.com/jito-foundation/jito-solana/releases/download/"$TAG"/solana-release-$TARGET.tar.bz2 ;; esac diff --git a/scripts/increment-cargo-version.sh b/scripts/increment-cargo-version.sh index 866f442874..41e1994ced 100755 --- a/scripts/increment-cargo-version.sh +++ b/scripts/increment-cargo-version.sh @@ -23,6 +23,8 @@ ignores=( .cargo target node_modules + jito-programs + anchor ) not_paths=() diff --git a/scripts/run.sh b/scripts/run.sh index 70994c921f..4a148dfed5 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -104,6 +104,10 @@ args=( --identity "$validator_identity" --vote-account "$validator_vote_account" --ledger "$ledgerDir" + --tip-payment-program-pubkey "T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt" + --tip-distribution-program-pubkey "4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7" + --merkle-root-upload-authority "$validator_identity" + --commission-bps 0 --gossip-port 8001 --full-rpc-api --rpc-port 8899 diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index fcc164c0e8..db2fdc8e74 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -18,7 +18,7 @@ program = [] default = [ "borsh", - "full", # functionality that is not compatible or needed for on-chain programs + "full", # functionality that is not compatible or needed for on-chain programs ] full = [ "byteorder", @@ -44,6 +44,9 @@ frozen-abi = [ ] [dependencies] +anchor-lang = { workspace = true } +assert_matches = { workspace = true, optional = true } +base64 = { workspace = true } bincode = { workspace = true } bitflags = { workspace = true, features = ["serde"] } borsh = { workspace = true, optional = true } diff --git a/sdk/src/bundle/mod.rs b/sdk/src/bundle/mod.rs new file mode 100644 index 0000000000..3c02a59f9f --- /dev/null +++ b/sdk/src/bundle/mod.rs @@ -0,0 +1,33 @@ +#![cfg(feature = "full")] + +use { + crate::transaction::{SanitizedTransaction, VersionedTransaction}, + digest::Digest, + itertools::Itertools, + sha2::Sha256, +}; + +#[derive(Debug, PartialEq, Default, Eq, Clone, Serialize, Deserialize)] +pub struct VersionedBundle { + pub transactions: Vec, +} + +#[derive(Clone, Debug)] +pub struct SanitizedBundle { + pub transactions: Vec, + pub bundle_id: String, +} + +pub fn derive_bundle_id(transactions: &[VersionedTransaction]) -> String { + let mut hasher = Sha256::new(); + hasher.update(transactions.iter().map(|tx| tx.signatures[0]).join(",")); + format!("{:x}", hasher.finalize()) +} + +pub fn derive_bundle_id_from_sanitized_transactions( + transactions: &[SanitizedTransaction], +) -> String { + let mut hasher = Sha256::new(); + hasher.update(transactions.iter().map(|tx| tx.signature()).join(",")); + format!("{:x}", hasher.finalize()) +} diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 95a9b880ff..4fc3561dce 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -60,6 +60,7 @@ pub use solana_program::{ pub use solana_program::{borsh, borsh0_10, borsh1}; pub mod account; pub mod account_utils; +pub mod bundle; pub mod client; pub mod commitment_config; pub mod compute_budget; diff --git a/send-transaction-service/Cargo.toml b/send-transaction-service/Cargo.toml index a69c366a35..9da9b4d405 100644 --- a/send-transaction-service/Cargo.toml +++ b/send-transaction-service/Cargo.toml @@ -14,6 +14,7 @@ crossbeam-channel = { workspace = true } log = { workspace = true } solana-client = { workspace = true } solana-connection-cache = { workspace = true } +solana-gossip = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-runtime = { workspace = true } @@ -23,6 +24,7 @@ solana-tpu-client = { workspace = true } [dev-dependencies] solana-logger = { workspace = true } solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } +solana-streamer = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/send-transaction-service/src/send_transaction_service.rs b/send-transaction-service/src/send_transaction_service.rs index 8cc21b1235..9403658206 100644 --- a/send-transaction-service/src/send_transaction_service.rs +++ b/send-transaction-service/src/send_transaction_service.rs @@ -4,6 +4,7 @@ use { log::*, solana_client::connection_cache::{ConnectionCache, Protocol}, solana_connection_cache::client_connection::ClientConnection as TpuConnection, + solana_gossip::cluster_info::ClusterInfo, solana_measure::measure::Measure, solana_runtime::{bank::Bank, bank_forks::BankForks}, solana_sdk::{ @@ -330,7 +331,7 @@ const SEND_TRANSACTION_METRICS_REPORT_RATE_MS: u64 = 5000; impl SendTransactionService { pub fn new( - tpu_address: SocketAddr, + cluster_info: Arc, bank_forks: &Arc>, leader_info: Option, receiver: Receiver, @@ -345,7 +346,7 @@ impl SendTransactionService { ..Config::default() }; Self::new_with_config( - tpu_address, + cluster_info, bank_forks, leader_info, receiver, @@ -356,7 +357,7 @@ impl SendTransactionService { } pub fn new_with_config( - tpu_address: SocketAddr, + cluster_info: Arc, bank_forks: &Arc>, leader_info: Option, receiver: Receiver, @@ -371,7 +372,7 @@ impl SendTransactionService { let leader_info_provider = Arc::new(Mutex::new(CurrentLeaderInfo::new(leader_info))); let receive_txn_thread = Self::receive_txn_thread( - tpu_address, + cluster_info.clone(), receiver, leader_info_provider.clone(), connection_cache.clone(), @@ -382,7 +383,7 @@ impl SendTransactionService { ); let retry_thread = Self::retry_thread( - tpu_address, + cluster_info, bank_forks.clone(), leader_info_provider, connection_cache.clone(), @@ -400,7 +401,7 @@ impl SendTransactionService { /// Thread responsible for receiving transactions from RPC clients. fn receive_txn_thread( - tpu_address: SocketAddr, + cluster_info: Arc, receiver: Receiver, leader_info_provider: Arc>>, connection_cache: Arc, @@ -461,6 +462,10 @@ impl SendTransactionService { stats .sent_transactions .fetch_add(transactions.len() as u64, Ordering::Relaxed); + let tpu_address = cluster_info + .my_contact_info() + .tpu(connection_cache.protocol()) + .unwrap(); Self::send_transactions_in_batch( &tpu_address, &transactions, @@ -506,7 +511,7 @@ impl SendTransactionService { /// Thread responsible for retrying transactions fn retry_thread( - tpu_address: SocketAddr, + cluster_info: Arc, bank_forks: Arc>, leader_info_provider: Arc>>, connection_cache: Arc, @@ -539,7 +544,10 @@ impl SendTransactionService { let bank_forks = bank_forks.read().unwrap(); (bank_forks.root_bank(), bank_forks.working_bank()) }; - + let tpu_address = cluster_info + .my_contact_info() + .tpu(connection_cache.protocol()) + .unwrap(); let _result = Self::process_transactions( &working_bank, &root_bank, @@ -824,27 +832,40 @@ mod test { super::*, crate::tpu_info::NullTpuInfo, crossbeam_channel::{bounded, unbounded}, + solana_gossip::contact_info::ContactInfo, solana_sdk::{ account::AccountSharedData, genesis_config::create_genesis_config, nonce::{self, state::DurableNonce}, pubkey::Pubkey, - signature::Signer, + signature::{Keypair, Signer}, system_program, system_transaction, + timing::timestamp, }, + solana_streamer::socket::SocketAddrSpace, std::ops::Sub, }; + fn new_test_cluster_info() -> Arc { + let keypair = Arc::new(Keypair::new()); + let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), timestamp()); + Arc::new(ClusterInfo::new( + contact_info, + keypair, + SocketAddrSpace::Unspecified, + )) + } + #[test] fn service_exit() { - let tpu_address = "127.0.0.1:0".parse().unwrap(); let bank = Bank::default_for_tests(); let bank_forks = BankForks::new_rw_arc(bank); let (sender, receiver) = unbounded(); let connection_cache = Arc::new(ConnectionCache::new("connection_cache_test")); + let cluster_info = new_test_cluster_info(); let send_transaction_service = SendTransactionService::new::( - tpu_address, + cluster_info, &bank_forks, None, receiver, @@ -860,7 +881,7 @@ mod test { #[test] fn validator_exit() { - let tpu_address = "127.0.0.1:0".parse().unwrap(); + let cluster_info = new_test_cluster_info(); let bank = Bank::default_for_tests(); let bank_forks = BankForks::new_rw_arc(bank); let (sender, receiver) = bounded(0); @@ -878,7 +899,7 @@ mod test { let exit = Arc::new(AtomicBool::new(false)); let connection_cache = Arc::new(ConnectionCache::new("connection_cache_test")); let _send_transaction_service = SendTransactionService::new::( - tpu_address, + cluster_info, &bank_forks, None, receiver, diff --git a/start b/start new file mode 100755 index 0000000000..c2f35e272a --- /dev/null +++ b/start @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eu + +SOLANA_CONFIG_DIR=./config + +mkdir -p $SOLANA_CONFIG_DIR +NDEBUG=1 ./multinode-demo/setup.sh +cargo run --release --bin solana-ledger-tool -- -l config/bootstrap-validator/ create-snapshot 0 +NDEBUG=1 ./multinode-demo/faucet.sh diff --git a/start_multi b/start_multi new file mode 100755 index 0000000000..66de0032dc --- /dev/null +++ b/start_multi @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -eu + +SOLANA_KEYGEN="cargo run --release --bin solana-keygen --" +SOLANA_CONFIG_DIR=./config + +if [[ ! -d $SOLANA_CONFIG_DIR ]]; then + echo "New Config! Generating Identities" + mkdir $SOLANA_CONFIG_DIR + $SOLANA_KEYGEN new --no-passphrase -so "$SOLANA_CONFIG_DIR"/a/identity.json + $SOLANA_KEYGEN new --no-passphrase -so "$SOLANA_CONFIG_DIR"/a/stake-account.json + $SOLANA_KEYGEN new --no-passphrase -so "$SOLANA_CONFIG_DIR"/a/vote-account.json + + $SOLANA_KEYGEN new --no-passphrase -so "$SOLANA_CONFIG_DIR"/b/identity.json + $SOLANA_KEYGEN new --no-passphrase -so "$SOLANA_CONFIG_DIR"/b/stake-account.json + $SOLANA_KEYGEN new --no-passphrase -so "$SOLANA_CONFIG_DIR"/b/vote-account.json +fi + +NDEBUG=1 ./multinode-demo/setup.sh \ + --bootstrap-validator \ + "$SOLANA_CONFIG_DIR"/a/identity.json \ + "$SOLANA_CONFIG_DIR"/a/vote-account.json \ + "$SOLANA_CONFIG_DIR"/a/stake-account.json \ + --bootstrap-validator \ + "$SOLANA_CONFIG_DIR"/b/identity.json \ + "$SOLANA_CONFIG_DIR"/b/vote-account.json \ + "$SOLANA_CONFIG_DIR"/b/stake-account.json + +cargo run --bin solana-ledger-tool -- -l config/bootstrap-validator/ create-snapshot 0 +NDEBUG=1 ./multinode-demo/faucet.sh diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index c62963c09e..4cf6c882d3 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -6,6 +6,7 @@ use { transaction_processing_callback::TransactionProcessingCallback, }, itertools::Itertools, + log::info, solana_compute_budget::compute_budget_processor::{ process_compute_budget_instructions, ComputeBudgetLimits, }, @@ -235,6 +236,10 @@ fn load_transaction_accounts( } else if let Some(account_override) = account_overrides.and_then(|overrides| overrides.get(key)) { + info!( + "loaded account from cache key: {:?} override: {:?}", + key, account_override + ); (account_override.data().len(), account_override.clone(), 0) } else if let Some(program) = (!instruction_account && !message.is_writable(i)) .then_some(()) diff --git a/svm/src/account_overrides.rs b/svm/src/account_overrides.rs index 8a205a798f..b92ba7d0ad 100644 --- a/svm/src/account_overrides.rs +++ b/svm/src/account_overrides.rs @@ -4,12 +4,16 @@ use { }; /// Encapsulates overridden accounts, typically used for transaction simulations -#[derive(Default)] +#[derive(Clone, Default, Debug)] pub struct AccountOverrides { accounts: HashMap, } impl AccountOverrides { + pub fn upsert_account_overrides(&mut self, other: AccountOverrides) { + self.accounts.extend(other.accounts); + } + /// Insert or remove an account with a given pubkey to/from the list of overrides. pub fn set_account(&mut self, pubkey: &Pubkey, account: Option) { match account { diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index 843060e22e..965ff80028 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -245,7 +245,8 @@ impl TransactionBatchProcessor { environment .rent_collector .unwrap_or(&RentCollector::default()), - &mut error_metrics + &mut error_metrics, + config.account_overrides )); let mut program_cache_time = Measure::start("program_cache"); @@ -387,6 +388,7 @@ impl TransactionBatchProcessor { fee_structure: &FeeStructure, rent_collector: &RentCollector, error_counters: &mut TransactionErrorMetrics, + account_overrides: Option<&AccountOverrides>, ) -> Vec { sanitized_txs .iter() @@ -402,6 +404,7 @@ impl TransactionBatchProcessor { fee_structure, rent_collector, error_counters, + account_overrides, ) }) }) @@ -420,6 +423,7 @@ impl TransactionBatchProcessor { fee_structure: &FeeStructure, rent_collector: &RentCollector, error_counters: &mut TransactionErrorMetrics, + account_overrides: Option<&AccountOverrides>, ) -> transaction::Result { let compute_budget_limits = process_compute_budget_instructions( message.program_instructions_iter(), @@ -430,8 +434,14 @@ impl TransactionBatchProcessor { })?; let fee_payer_address = message.fee_payer(); - let Some(mut fee_payer_account) = callbacks.get_account_shared_data(fee_payer_address) - else { + let maybe_account = if let Some(account_override) = + account_overrides.and_then(|overrides| overrides.get(fee_payer_address)) + { + Some(account_override.clone()) + } else { + callbacks.get_account_shared_data(fee_payer_address) + }; + let Some(mut fee_payer_account) = maybe_account else { error_counters.account_not_found += 1; return Err(TransactionError::AccountNotFound); }; @@ -2004,6 +2014,7 @@ mod tests { &FeeStructure::default(), &rent_collector, &mut error_counters, + None, ); let post_validation_fee_payer_account = { @@ -2076,6 +2087,7 @@ mod tests { &FeeStructure::default(), &rent_collector, &mut error_counters, + None, ); let post_validation_fee_payer_account = { @@ -2123,6 +2135,7 @@ mod tests { &FeeStructure::default(), &RentCollector::default(), &mut error_counters, + None, ); assert_eq!(error_counters.account_not_found, 1); @@ -2155,6 +2168,7 @@ mod tests { &FeeStructure::default(), &RentCollector::default(), &mut error_counters, + None, ); assert_eq!(error_counters.insufficient_funds, 1); @@ -2191,6 +2205,7 @@ mod tests { &FeeStructure::default(), &rent_collector, &mut error_counters, + None, ); assert_eq!( @@ -2225,6 +2240,7 @@ mod tests { &FeeStructure::default(), &RentCollector::default(), &mut error_counters, + None, ); assert_eq!(error_counters.invalid_account_for_fee, 1); @@ -2256,6 +2272,7 @@ mod tests { &FeeStructure::default(), &RentCollector::default(), &mut error_counters, + None, ); assert_eq!(error_counters.invalid_compute_budget, 1); @@ -2317,6 +2334,7 @@ mod tests { &FeeStructure::default(), &rent_collector, &mut error_counters, + None, ); let post_validation_fee_payer_account = { @@ -2374,10 +2392,13 @@ mod tests { &FeeStructure::default(), &rent_collector, &mut error_counters, + None, ); assert_eq!(error_counters.insufficient_funds, 1); assert_eq!(result, Err(TransactionError::InsufficientFundsForFee)); } } + + // TODO (LB): test loading invalid fee payer from cache w/ valid (and invalid) fees } diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 39616c1fdd..4c21c81c2e 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -1025,6 +1025,7 @@ impl TestValidator { config.tpu_enable_udp, 32, // max connections per IpAddr per minute for test config.admin_rpc_service_post_init.clone(), + None, )?); // Needed to avoid panics in `solana-responder-gossip` in tests that create a number of diff --git a/timings/src/lib.rs b/timings/src/lib.rs index efc918c71d..85f0ceb594 100644 --- a/timings/src/lib.rs +++ b/timings/src/lib.rs @@ -10,7 +10,7 @@ use { }, }; -#[derive(Default, Debug, PartialEq, Eq)] +#[derive(Clone, Default, Debug, PartialEq, Eq)] pub struct ProgramTiming { pub accumulated_us: u64, pub accumulated_units: u64, @@ -58,6 +58,7 @@ pub enum ExecuteTimingType { CheckBlockLimitsUs, } +#[derive(Clone)] pub struct Metrics([u64; ExecuteTimingType::CARDINALITY]); impl Index for Metrics { @@ -300,7 +301,7 @@ eager_macro_rules! { $eager_1 } } -#[derive(Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct ExecuteTimings { pub metrics: Metrics, pub details: ExecuteDetailsTimings, @@ -324,9 +325,21 @@ impl ExecuteTimings { None => debug_assert!(idx < ExecuteTimingType::CARDINALITY, "Index out of bounds"), } } + + pub fn accumulate_execute_units_and_time(&self) -> (u64, u64) { + self.details + .per_program_timings + .values() + .fold((0, 0), |(units, times), program_timings| { + ( + units.saturating_add(program_timings.accumulated_units), + times.saturating_add(program_timings.accumulated_us), + ) + }) + } } -#[derive(Default, Debug)] +#[derive(Clone, Default, Debug)] pub struct ExecuteProcessInstructionTimings { pub total_us: u64, pub verify_caller_us: u64, @@ -346,7 +359,7 @@ impl ExecuteProcessInstructionTimings { } } -#[derive(Default, Debug)] +#[derive(Clone, Default, Debug)] pub struct ExecuteAccessoryTimings { pub feature_set_clone_us: u64, pub get_executors_us: u64, @@ -366,7 +379,7 @@ impl ExecuteAccessoryTimings { } } -#[derive(Default, Debug, PartialEq, Eq)] +#[derive(Clone, Default, Debug, PartialEq, Eq)] pub struct ExecuteDetailsTimings { pub serialize_us: u64, pub create_vm_us: u64, diff --git a/tip-distributor/Cargo.toml b/tip-distributor/Cargo.toml new file mode 100644 index 0000000000..76682d220a --- /dev/null +++ b/tip-distributor/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "solana-tip-distributor" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +description = "Collection of binaries used to distribute MEV rewards to delegators and validators." +publish = false + +[dependencies] +anchor-lang = { workspace = true } +clap = { version = "4.1.11", features = ["derive", "env"] } +crossbeam-channel = { workspace = true } +env_logger = { workspace = true } +futures = { workspace = true } +gethostname = { workspace = true } +im = { workspace = true } +itertools = { workspace = true } +jito-tip-distribution = { workspace = true } +jito-tip-payment = { workspace = true } +log = { workspace = true } +num-traits = { workspace = true } +rand = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +solana-accounts-db = { workspace = true } +solana-client = { workspace = true } +solana-genesis-utils = { workspace = true } +solana-ledger = { workspace = true } +solana-measure = { workspace = true } +solana-merkle-tree = { workspace = true } +solana-metrics = { workspace = true } +solana-program = { workspace = true } +solana-program-runtime = { workspace = true } +solana-rpc-client-api = { workspace = true } +solana-runtime = { workspace = true } +solana-sdk = { workspace = true } +solana-stake-program = { workspace = true } +solana-transaction-status = { workspace = true } +solana-vote = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[dev-dependencies] +solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } +solana-sdk = { workspace = true, features = ["dev-context-only-utils"] } + +[[bin]] +name = "solana-stake-meta-generator" +path = "src/bin/stake-meta-generator.rs" + +[[bin]] +name = "solana-merkle-root-generator" +path = "src/bin/merkle-root-generator.rs" + +[[bin]] +name = "solana-merkle-root-uploader" +path = "src/bin/merkle-root-uploader.rs" + +[[bin]] +name = "solana-claim-mev-tips" +path = "src/bin/claim-mev-tips.rs" diff --git a/tip-distributor/README.md b/tip-distributor/README.md new file mode 100644 index 0000000000..fec682879a --- /dev/null +++ b/tip-distributor/README.md @@ -0,0 +1,52 @@ +# Tip Distributor +This library and collection of binaries are responsible for generating and uploading merkle roots to the on-chain +tip-distribution program found [here](https://github.com/jito-foundation/jito-programs/blob/submodule/tip-payment/programs/tip-distribution/src/lib.rs). + +## Background +Each individual validator is assigned a new PDA per epoch where their share of tips, in lamports, will be stored. +At the end of the epoch it's expected that validators take a commission and then distribute the rest of the funds +to their delegators such that delegators receive rewards proportional to their respective delegations. The distribution +mechanism is via merkle proofs similar to how airdrops work. + +The merkle roots are calculated off-chain and uploaded to the validator's **TipDistributionAccount** PDA. Validators may +elect an account to upload the merkle roots on their behalf. Once uploaded, users can invoke the **claim** instruction +and receive the rewards they're entitled to. Once all funds are claimed by users the validator can close the account and +refunded the rent. + +## Scripts + +### stake-meta-generator + +This script generates a JSON file identifying individual stake delegations to a validator, along with amount of lamports +in each validator's **TipDistributionAccount**. All validators will be contained in the JSON list, regardless of whether +the validator is a participant in the system; participant being indicative of running the jito-solana client to accept tips +having initialized a **TipDistributionAccount** PDA account for the epoch. + +One edge case that we've taken into account is the last validator in an epoch N receives tips but those tips don't get transferred +out into the PDA until some slot in epoch N + 1. Due to this we cannot rely on the bank's state at epoch N for lamports amount +in the PDAs. We use the bank solely to take a snapshot of delegations, but an RPC node to fetch the PDA lamports for more up-to-date data. + +### merkle-root-generator +This script accepts a path to the above JSON file as one of its arguments, and generates a merkle-root into a JSON file. + +### merkle-root-uploader +Uploads the root on-chain. + +### claim-mev-tips +This reads the file outputted by `merkle-root-generator` and finds all eligible accounts to receive mev tips. Transactions +are created and sent to the RPC server. + + +## How it works? +In order to use this library as the merkle root creator one must follow the following steps: +1. Download a ledger snapshot containing the slot of interest, i.e. the last slot in an epoch. The Solana foundation has snapshots that can be found [here](https://console.cloud.google.com/storage/browser/mainnet-beta-ledger-us-ny5). +2. Download the snapshot onto your worker machine (where this script will run). +3. Run `solana-ledger-tool -l ${PATH_TO_LEDGER} create-snapshot ${YOUR_SLOT} ${WHERE_TO_CREATE_SNAPSHOT}` + 1. The snapshot created at `${WHERE_TO_CREATE_SNAPSHOT}` will have the highest slot of `${YOUR_SLOT}`, assuming you downloaded the correct snapshot. +4. Run `stake-meta-generator --ledger-path ${WHERE_TO_CREATE_SNAPSHOT} --tip-distribution-program-id ${PUBKEY} --out-path ${JSON_OUT_PATH} --snapshot-slot ${SLOT} --rpc-url ${URL}` + 1. Note: `${WHERE_TO_CREATE_SNAPSHOT}` must be the same in steps 3 & 4. +5. Run `merkle-root-generator --stake-meta-coll-path ${STAKE_META_COLLECTION_JSON} --rpc-url ${URL} --out-path ${MERKLE_ROOT_PATH}` +6. Run `merkle-root-uploader --out-path ${MERKLE_ROOT_PATH} --keypair-path ${KEYPAIR_PATH} --rpc-url ${URL} --tip-distribution-program-id ${PROGRAM_ID}` +7. Run `solana-claim-mev-tips --merkle-trees-path /solana/ledger/autosnapshot/merkle-tree-221615999.json --rpc-url ${URL} --tip-distribution-program-id ${PROGRAM_ID} --keypair-path ${KEYPAIR_PATH}` + +Voila! diff --git a/tip-distributor/src/bin/claim-mev-tips.rs b/tip-distributor/src/bin/claim-mev-tips.rs new file mode 100644 index 0000000000..dd57db9231 --- /dev/null +++ b/tip-distributor/src/bin/claim-mev-tips.rs @@ -0,0 +1,190 @@ +//! This binary claims MEV tips. +use { + clap::Parser, + futures::future::join_all, + gethostname::gethostname, + log::*, + solana_metrics::{datapoint_error, datapoint_info, set_host_id}, + solana_sdk::{ + pubkey::Pubkey, + signature::{read_keypair_file, Keypair}, + }, + solana_tip_distributor::{ + claim_mev_workflow::{claim_mev_tips, ClaimMevError}, + read_json_from_file, + reclaim_rent_workflow::reclaim_rent, + GeneratedMerkleTreeCollection, + }, + std::{ + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, + }, +}; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Path to JSON file containing the [GeneratedMerkleTreeCollection] object. + #[arg(long, env)] + merkle_trees_path: PathBuf, + + /// RPC to send transactions through + #[arg(long, env, default_value = "http://localhost:8899")] + rpc_url: String, + + /// Tip distribution program ID + #[arg(long, env)] + tip_distribution_program_id: Pubkey, + + /// Path to keypair + #[arg(long, env)] + keypair_path: PathBuf, + + /// Limits how long before send loop runs before stopping + #[arg(long, env, default_value_t = 60 * 60)] + max_retry_duration_secs: u64, + + /// Specifies whether to reclaim any rent. + #[arg(long, env, default_value_t = true)] + should_reclaim_rent: bool, + + /// Specifies whether to reclaim rent on behalf of validators from respective TDAs. + #[arg(long, env)] + should_reclaim_tdas: bool, + + /// The price to pay for priority fee + #[arg(long, env, default_value_t = 1)] + micro_lamports: u64, +} + +async fn start_mev_claim_process( + merkle_trees: GeneratedMerkleTreeCollection, + rpc_url: String, + tip_distribution_program_id: Pubkey, + signer: Arc, + max_loop_duration: Duration, + micro_lamports: u64, +) -> Result<(), ClaimMevError> { + let start = Instant::now(); + + match claim_mev_tips( + &merkle_trees, + rpc_url, + tip_distribution_program_id, + signer, + max_loop_duration, + micro_lamports, + ) + .await + { + Err(e) => { + datapoint_error!( + "claim_mev_workflow-claim_error", + ("epoch", merkle_trees.epoch, i64), + ("error", 1, i64), + ("err_str", e.to_string(), String), + ("elapsed_us", start.elapsed().as_micros(), i64), + ); + Err(e) + } + Ok(()) => { + datapoint_info!( + "claim_mev_workflow-claim_completion", + ("epoch", merkle_trees.epoch, i64), + ("elapsed_us", start.elapsed().as_micros(), i64), + ); + Ok(()) + } + } +} + +async fn start_rent_claim( + rpc_url: String, + tip_distribution_program_id: Pubkey, + signer: Arc, + max_loop_duration: Duration, + should_reclaim_tdas: bool, + micro_lamports: u64, + epoch: u64, +) -> Result<(), ClaimMevError> { + let start = Instant::now(); + match reclaim_rent( + rpc_url, + tip_distribution_program_id, + signer, + max_loop_duration, + should_reclaim_tdas, + micro_lamports, + ) + .await + { + Err(e) => { + datapoint_error!( + "claim_mev_workflow-reclaim_rent_error", + ("epoch", epoch, i64), + ("error", 1, i64), + ("err_str", e.to_string(), String), + ("elapsed_us", start.elapsed().as_micros(), i64), + ); + Err(e) + } + Ok(()) => { + datapoint_info!( + "claim_mev_workflow-reclaim_rent_completion", + ("epoch", epoch, i64), + ("elapsed_us", start.elapsed().as_micros(), i64), + ); + Ok(()) + } + } +} + +#[tokio::main] +async fn main() -> Result<(), ClaimMevError> { + env_logger::init(); + + gethostname() + .into_string() + .map(set_host_id) + .expect("set hostname"); + + let args: Args = Args::parse(); + let keypair = Arc::new(read_keypair_file(&args.keypair_path).expect("read keypair file")); + let merkle_trees: GeneratedMerkleTreeCollection = + read_json_from_file(&args.merkle_trees_path).expect("read GeneratedMerkleTreeCollection"); + let max_loop_duration = Duration::from_secs(args.max_retry_duration_secs); + + info!( + "Starting to claim mev tips for epoch: {}", + merkle_trees.epoch + ); + let epoch = merkle_trees.epoch; + + let mut futs = vec![]; + futs.push(tokio::spawn(start_mev_claim_process( + merkle_trees, + args.rpc_url.clone(), + args.tip_distribution_program_id, + keypair.clone(), + max_loop_duration, + args.micro_lamports, + ))); + if args.should_reclaim_rent { + futs.push(tokio::spawn(start_rent_claim( + args.rpc_url.clone(), + args.tip_distribution_program_id, + keypair.clone(), + max_loop_duration, + args.should_reclaim_tdas, + args.micro_lamports, + epoch, + ))); + } + let results = join_all(futs).await; + solana_metrics::flush(); // sometimes last datapoint doesn't get emitted. this increases likelihood. + for r in results { + r.map_err(|e| ClaimMevError::UncaughtError { e: e.to_string() })??; + } + Ok(()) +} diff --git a/tip-distributor/src/bin/merkle-root-generator.rs b/tip-distributor/src/bin/merkle-root-generator.rs new file mode 100644 index 0000000000..9f2d0f9a4e --- /dev/null +++ b/tip-distributor/src/bin/merkle-root-generator.rs @@ -0,0 +1,34 @@ +//! This binary generates a merkle tree for each [TipDistributionAccount]; they are derived +//! using a user provided [StakeMetaCollection] JSON file. + +use { + clap::Parser, log::*, + solana_tip_distributor::merkle_root_generator_workflow::generate_merkle_root, + std::path::PathBuf, +}; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Path to JSON file containing the [StakeMetaCollection] object. + #[arg(long, env)] + stake_meta_coll_path: PathBuf, + + /// RPC to send transactions through. Used to validate what's being claimed is equal to TDA balance minus rent. + #[arg(long, env)] + rpc_url: String, + + /// Path to JSON file to get populated with tree node data. + #[arg(long, env)] + out_path: PathBuf, +} + +fn main() { + env_logger::init(); + info!("Starting merkle-root-generator workflow..."); + + let args: Args = Args::parse(); + generate_merkle_root(&args.stake_meta_coll_path, &args.out_path, &args.rpc_url) + .expect("merkle tree produced"); + info!("saved merkle roots to {:?}", args.stake_meta_coll_path); +} diff --git a/tip-distributor/src/bin/merkle-root-uploader.rs b/tip-distributor/src/bin/merkle-root-uploader.rs new file mode 100644 index 0000000000..9000ce66d0 --- /dev/null +++ b/tip-distributor/src/bin/merkle-root-uploader.rs @@ -0,0 +1,54 @@ +use { + clap::Parser, log::info, solana_sdk::pubkey::Pubkey, + solana_tip_distributor::merkle_root_upload_workflow::upload_merkle_root, std::path::PathBuf, +}; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Path to JSON file containing the [StakeMetaCollection] object. + #[arg(long, env)] + merkle_root_path: PathBuf, + + /// The path to the keypair used to sign and pay for the `upload_merkle_root` transactions. + #[arg(long, env)] + keypair_path: PathBuf, + + /// The RPC to send transactions to. + #[arg(long, env)] + rpc_url: String, + + /// Tip distribution program ID + #[arg(long, env)] + tip_distribution_program_id: Pubkey, + + /// Rate-limits the maximum number of requests per RPC connection + #[arg(long, env, default_value_t = 100)] + max_concurrent_rpc_get_reqs: usize, + + /// Number of transactions to send to RPC at a time. + #[arg(long, env, default_value_t = 64)] + txn_send_batch_size: usize, +} + +fn main() { + env_logger::init(); + + let args: Args = Args::parse(); + + info!("starting merkle root uploader..."); + if let Err(e) = upload_merkle_root( + &args.merkle_root_path, + &args.keypair_path, + &args.rpc_url, + &args.tip_distribution_program_id, + args.max_concurrent_rpc_get_reqs, + args.txn_send_batch_size, + ) { + panic!("failed to upload merkle roots: {:?}", e); + } + info!( + "uploaded merkle roots from file {:?}", + args.merkle_root_path + ); +} diff --git a/tip-distributor/src/bin/stake-meta-generator.rs b/tip-distributor/src/bin/stake-meta-generator.rs new file mode 100644 index 0000000000..be7993be02 --- /dev/null +++ b/tip-distributor/src/bin/stake-meta-generator.rs @@ -0,0 +1,67 @@ +//! This binary is responsible for generating a JSON file that contains meta-data about stake +//! & delegations given a ledger snapshot directory. The JSON file is structured as an array +//! of [StakeMeta] objects. + +use { + clap::Parser, + log::*, + solana_sdk::{clock::Slot, pubkey::Pubkey}, + solana_tip_distributor::{self, stake_meta_generator_workflow::generate_stake_meta}, + std::{ + fs::{self}, + path::PathBuf, + process::exit, + }, +}; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Ledger path, where you created the snapshot. + #[arg(long, env, value_parser = Args::ledger_path_parser)] + ledger_path: PathBuf, + + /// The tip-distribution program id. + #[arg(long, env)] + tip_distribution_program_id: Pubkey, + + /// The tip-payment program id. + #[arg(long, env)] + tip_payment_program_id: Pubkey, + + /// Path to JSON file populated with the [StakeMetaCollection] object. + #[arg(long, env)] + out_path: String, + + /// The expected snapshot slot. + #[arg(long, env)] + snapshot_slot: Slot, +} + +impl Args { + fn ledger_path_parser(ledger_path: &str) -> Result { + Ok(fs::canonicalize(ledger_path).unwrap_or_else(|err| { + error!("Unable to access ledger path '{}': {}", ledger_path, err); + exit(1); + })) + } +} + +fn main() { + env_logger::init(); + info!("Starting stake-meta-generator..."); + + let args: Args = Args::parse(); + + if let Err(e) = generate_stake_meta( + &args.ledger_path, + &args.snapshot_slot, + &args.tip_distribution_program_id, + &args.out_path, + &args.tip_payment_program_id, + ) { + error!("error producing stake-meta: {:?}", e); + } else { + info!("produced stake meta"); + } +} diff --git a/tip-distributor/src/claim_mev_workflow.rs b/tip-distributor/src/claim_mev_workflow.rs new file mode 100644 index 0000000000..62b0b6fb02 --- /dev/null +++ b/tip-distributor/src/claim_mev_workflow.rs @@ -0,0 +1,398 @@ +use { + crate::{send_until_blockhash_expires, GeneratedMerkleTreeCollection}, + anchor_lang::{AccountDeserialize, InstructionData, ToAccountMetas}, + itertools::Itertools, + jito_tip_distribution::state::{ClaimStatus, Config, TipDistributionAccount}, + log::{error, info, warn}, + rand::{prelude::SliceRandom, thread_rng}, + solana_client::nonblocking::rpc_client::RpcClient, + solana_metrics::datapoint_info, + solana_program::{ + fee_calculator::DEFAULT_TARGET_LAMPORTS_PER_SIGNATURE, native_token::LAMPORTS_PER_SOL, + system_program, + }, + solana_rpc_client_api::config::RpcSimulateTransactionConfig, + solana_sdk::{ + account::Account, + commitment_config::CommitmentConfig, + compute_budget::ComputeBudgetInstruction, + instruction::Instruction, + pubkey::Pubkey, + signature::{Keypair, Signer}, + transaction::Transaction, + }, + std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, + }, + thiserror::Error, +}; + +#[derive(Error, Debug)] +pub enum ClaimMevError { + #[error(transparent)] + IoError(#[from] std::io::Error), + + #[error(transparent)] + JsonError(#[from] serde_json::Error), + + #[error(transparent)] + AnchorError(anchor_lang::error::Error), + + #[error(transparent)] + RpcError(#[from] solana_rpc_client_api::client_error::Error), + + #[error("Expected to have at least {desired_balance} lamports in {payer:?}. Current balance is {start_balance} lamports. Deposit {sol_to_deposit} SOL to continue.")] + InsufficientBalance { + desired_balance: u64, + payer: Pubkey, + start_balance: u64, + sol_to_deposit: u64, + }, + + #[error("Not finished with job, transactions left {transactions_left}")] + NotFinished { transactions_left: usize }, + + #[error("UncaughtError {e:?}")] + UncaughtError { e: String }, +} + +pub async fn get_claim_transactions_for_valid_unclaimed( + rpc_client: &RpcClient, + merkle_trees: &GeneratedMerkleTreeCollection, + tip_distribution_program_id: Pubkey, + micro_lamports: u64, + payer_pubkey: Pubkey, +) -> Result, ClaimMevError> { + let tree_nodes = merkle_trees + .generated_merkle_trees + .iter() + .flat_map(|tree| &tree.tree_nodes) + .collect_vec(); + + info!( + "reading tip distribution related accounts for epoch {}", + merkle_trees.epoch + ); + + let start = Instant::now(); + + let tda_pubkeys = merkle_trees + .generated_merkle_trees + .iter() + .map(|tree| tree.tip_distribution_account) + .collect_vec(); + let tdas: HashMap = crate::get_batched_accounts(rpc_client, &tda_pubkeys) + .await? + .into_iter() + .filter_map(|(pubkey, a)| Some((pubkey, a?))) + .collect(); + + let claimant_pubkeys = tree_nodes + .iter() + .map(|tree_node| tree_node.claimant) + .collect_vec(); + let claimants: HashMap = + crate::get_batched_accounts(rpc_client, &claimant_pubkeys) + .await? + .into_iter() + .filter_map(|(pubkey, a)| Some((pubkey, a?))) + .collect(); + + let claim_status_pubkeys = tree_nodes + .iter() + .map(|tree_node| tree_node.claim_status_pubkey) + .collect_vec(); + let claim_statuses: HashMap = + crate::get_batched_accounts(rpc_client, &claim_status_pubkeys) + .await? + .into_iter() + .filter_map(|(pubkey, a)| Some((pubkey, a?))) + .collect(); + + let elapsed_us = start.elapsed().as_micros(); + + // can be helpful for determining mismatch in state between requested and read + datapoint_info!( + "claim_mev-get_claim_transactions_account_data", + ("elapsed_us", elapsed_us, i64), + ("tdas", tda_pubkeys.len(), i64), + ("tdas_onchain", tdas.len(), i64), + ("claimants", claimant_pubkeys.len(), i64), + ("claimants_onchain", claimants.len(), i64), + ("claim_statuses", claim_status_pubkeys.len(), i64), + ("claim_statuses_onchain", claim_statuses.len(), i64), + ); + + let transactions = build_mev_claim_transactions( + tip_distribution_program_id, + merkle_trees, + tdas, + claimants, + claim_statuses, + micro_lamports, + payer_pubkey, + ); + + Ok(transactions) +} + +pub async fn claim_mev_tips( + merkle_trees: &GeneratedMerkleTreeCollection, + rpc_url: String, + tip_distribution_program_id: Pubkey, + keypair: Arc, + max_loop_duration: Duration, + micro_lamports: u64, +) -> Result<(), ClaimMevError> { + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url, + Duration::from_secs(300), + CommitmentConfig::confirmed(), + ); + + let start = Instant::now(); + while start.elapsed() <= max_loop_duration { + let mut all_claim_transactions = get_claim_transactions_for_valid_unclaimed( + &rpc_client, + merkle_trees, + tip_distribution_program_id, + micro_lamports, + keypair.pubkey(), + ) + .await?; + + datapoint_info!( + "claim_mev_tips-send_summary", + ("claim_transactions_left", all_claim_transactions.len(), i64), + ); + + if all_claim_transactions.is_empty() { + return Ok(()); + } + + all_claim_transactions.shuffle(&mut thread_rng()); + let transactions: Vec<_> = all_claim_transactions.into_iter().take(10_000).collect(); + + // only check balance for the ones we need to currently send since reclaim rent running in parallel + if let Some((start_balance, desired_balance, sol_to_deposit)) = + is_sufficient_balance(&keypair.pubkey(), &rpc_client, transactions.len() as u64).await + { + return Err(ClaimMevError::InsufficientBalance { + desired_balance, + payer: keypair.pubkey(), + start_balance, + sol_to_deposit, + }); + } + + let blockhash = rpc_client.get_latest_blockhash().await?; + let _ = send_until_blockhash_expires(&rpc_client, transactions, blockhash, &keypair).await; + } + + let transactions = get_claim_transactions_for_valid_unclaimed( + &rpc_client, + merkle_trees, + tip_distribution_program_id, + micro_lamports, + keypair.pubkey(), + ) + .await?; + if transactions.is_empty() { + return Ok(()); + } + + // if more transactions left, we'll simulate them all to make sure its not an uncaught error + let mut is_error = false; + let mut error_str = String::new(); + for tx in &transactions { + match rpc_client + .simulate_transaction_with_config( + tx, + RpcSimulateTransactionConfig { + sig_verify: false, + replace_recent_blockhash: true, + commitment: Some(CommitmentConfig::processed()), + ..RpcSimulateTransactionConfig::default() + }, + ) + .await + { + Ok(_) => {} + Err(e) => { + error_str = e.to_string(); + is_error = true; + + match e.get_transaction_error() { + None => { + break; + } + Some(e) => { + warn!("transaction error. tx: {:?} error: {:?}", tx, e); + break; + } + } + } + } + } + + if is_error { + Err(ClaimMevError::UncaughtError { e: error_str }) + } else { + Err(ClaimMevError::NotFinished { + transactions_left: transactions.len(), + }) + } +} + +/// Returns a list of claim transactions for valid, unclaimed MEV tips +/// A valid, unclaimed transaction consists of the following: +/// - there must be lamports to claim for the tip distribution account. +/// - there must be a merkle root. +/// - the claimant (typically a stake account) must exist. +/// - the claimant (typically a stake account) must have a non-zero amount of tips to claim +/// - the claimant must have enough lamports post-claim to be rent-exempt. +/// - note: there aren't any rent exempt accounts on solana mainnet anymore. +/// - it must not have already been claimed. +fn build_mev_claim_transactions( + tip_distribution_program_id: Pubkey, + merkle_trees: &GeneratedMerkleTreeCollection, + tdas: HashMap, + claimants: HashMap, + claim_status: HashMap, + micro_lamports: u64, + payer_pubkey: Pubkey, +) -> Vec { + let tip_distribution_accounts: HashMap = tdas + .iter() + .filter_map(|(pubkey, account)| { + Some(( + *pubkey, + TipDistributionAccount::try_deserialize(&mut account.data.as_slice()).ok()?, + )) + }) + .collect(); + + let claim_statuses: HashMap = claim_status + .iter() + .filter_map(|(pubkey, account)| { + Some(( + *pubkey, + ClaimStatus::try_deserialize(&mut account.data.as_slice()).ok()?, + )) + }) + .collect(); + + datapoint_info!( + "build_mev_claim_transactions", + ( + "tip_distribution_accounts", + tip_distribution_accounts.len(), + i64 + ), + ("claim_statuses", claim_statuses.len(), i64), + ); + + let tip_distribution_config = + Pubkey::find_program_address(&[Config::SEED], &tip_distribution_program_id).0; + + let mut instructions = Vec::with_capacity(claimants.len()); + for tree in &merkle_trees.generated_merkle_trees { + if tree.max_total_claim == 0 { + continue; + } + + // if unwrap panics, there's a bug in the merkle tree code because the merkle tree code relies on the state + // of the chain to claim. + let tip_distribution_account = tip_distribution_accounts + .get(&tree.tip_distribution_account) + .unwrap(); + + // can continue here, as there might be tip distribution accounts this account doesn't upload for + if tip_distribution_account.merkle_root.is_none() { + continue; + } + + for node in &tree.tree_nodes { + // doesn't make sense to claim for claimants that don't exist anymore + // can't claim for something already claimed + // don't need to claim for claimants that get 0 MEV + if !claimants.contains_key(&node.claimant) + || claim_statuses.contains_key(&node.claim_status_pubkey) + || node.amount == 0 + { + continue; + } + + instructions.push(Instruction { + program_id: tip_distribution_program_id, + data: jito_tip_distribution::instruction::Claim { + proof: node.proof.clone().unwrap(), + amount: node.amount, + bump: node.claim_status_bump, + } + .data(), + accounts: jito_tip_distribution::accounts::Claim { + config: tip_distribution_config, + tip_distribution_account: tree.tip_distribution_account, + claimant: node.claimant, + claim_status: node.claim_status_pubkey, + payer: payer_pubkey, + system_program: system_program::id(), + } + .to_account_metas(None), + }); + } + } + + // TODO (LB): see if we can do >1 claim here + let transactions: Vec = instructions + .into_iter() + .map(|claim_ix| { + let priority_fee_ix = ComputeBudgetInstruction::set_compute_unit_price(micro_lamports); + Transaction::new_with_payer(&[priority_fee_ix, claim_ix], Some(&payer_pubkey)) + }) + .collect(); + + transactions +} + +/// heuristic to make sure we have enough funds to cover the rent costs if epoch has many validators +/// If insufficient funds, returns start balance, desired balance, and amount of sol to deposit +async fn is_sufficient_balance( + payer: &Pubkey, + rpc_client: &RpcClient, + instruction_count: u64, +) -> Option<(u64, u64, u64)> { + let start_balance = rpc_client + .get_balance(payer) + .await + .expect("Failed to get starting balance"); + // most amounts are for 0 lamports. had 1736 non-zero claims out of 164742 + let min_rent_per_claim = rpc_client + .get_minimum_balance_for_rent_exemption(ClaimStatus::SIZE) + .await + .expect("Failed to calculate min rent"); + let desired_balance = instruction_count + .checked_mul( + min_rent_per_claim + .checked_add(DEFAULT_TARGET_LAMPORTS_PER_SIGNATURE) + .unwrap(), + ) + .unwrap(); + if start_balance < desired_balance { + let sol_to_deposit = desired_balance + .checked_sub(start_balance) + .unwrap() + .checked_add(LAMPORTS_PER_SOL) + .unwrap() + .checked_sub(1) + .unwrap() + .checked_div(LAMPORTS_PER_SOL) + .unwrap(); // rounds up to nearest sol + Some((start_balance, desired_balance, sol_to_deposit)) + } else { + None + } +} diff --git a/tip-distributor/src/lib.rs b/tip-distributor/src/lib.rs new file mode 100644 index 0000000000..8ee6b50f5d --- /dev/null +++ b/tip-distributor/src/lib.rs @@ -0,0 +1,1062 @@ +pub mod claim_mev_workflow; +pub mod merkle_root_generator_workflow; +pub mod merkle_root_upload_workflow; +pub mod reclaim_rent_workflow; +pub mod stake_meta_generator_workflow; + +use { + crate::{ + merkle_root_generator_workflow::MerkleRootGeneratorError, + stake_meta_generator_workflow::StakeMetaGeneratorError::CheckedMathError, + }, + anchor_lang::Id, + jito_tip_distribution::{ + program::JitoTipDistribution, + state::{ClaimStatus, TipDistributionAccount}, + }, + jito_tip_payment::{ + Config, CONFIG_ACCOUNT_SEED, TIP_ACCOUNT_SEED_0, TIP_ACCOUNT_SEED_1, TIP_ACCOUNT_SEED_2, + TIP_ACCOUNT_SEED_3, TIP_ACCOUNT_SEED_4, TIP_ACCOUNT_SEED_5, TIP_ACCOUNT_SEED_6, + TIP_ACCOUNT_SEED_7, + }, + log::*, + serde::{de::DeserializeOwned, Deserialize, Serialize}, + solana_client::{ + nonblocking::rpc_client::RpcClient, + rpc_client::{RpcClient as SyncRpcClient, SerializableTransaction}, + }, + solana_merkle_tree::MerkleTree, + solana_metrics::{datapoint_error, datapoint_warn}, + solana_program::{ + instruction::InstructionError, + rent::{ + ACCOUNT_STORAGE_OVERHEAD, DEFAULT_EXEMPTION_THRESHOLD, DEFAULT_LAMPORTS_PER_BYTE_YEAR, + }, + }, + solana_rpc_client_api::{ + client_error::{Error, ErrorKind}, + config::RpcSendTransactionConfig, + request::{RpcError, RpcResponseErrorData, MAX_MULTIPLE_ACCOUNTS}, + response::RpcSimulateTransactionResult, + }, + solana_sdk::{ + account::{Account, AccountSharedData, ReadableAccount}, + clock::Slot, + commitment_config::{CommitmentConfig, CommitmentLevel}, + hash::{Hash, Hasher}, + pubkey::Pubkey, + signature::{Keypair, Signature}, + stake_history::Epoch, + transaction::{ + Transaction, + TransactionError::{self}, + }, + }, + solana_transaction_status::TransactionStatus, + std::{ + collections::{HashMap, HashSet}, + fs::File, + io::BufReader, + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, + }, + tokio::{sync::Semaphore, time::sleep}, +}; + +#[derive(Clone, Deserialize, Serialize, Debug)] +pub struct GeneratedMerkleTreeCollection { + pub generated_merkle_trees: Vec, + pub bank_hash: String, + pub epoch: Epoch, + pub slot: Slot, +} + +#[derive(Clone, Eq, Debug, Hash, PartialEq, Deserialize, Serialize)] +pub struct GeneratedMerkleTree { + #[serde(with = "pubkey_string_conversion")] + pub tip_distribution_account: Pubkey, + #[serde(with = "pubkey_string_conversion")] + pub merkle_root_upload_authority: Pubkey, + pub merkle_root: Hash, + pub tree_nodes: Vec, + pub max_total_claim: u64, + pub max_num_nodes: u64, +} + +pub struct TipPaymentPubkeys { + config_pda: Pubkey, + tip_pdas: Vec, +} + +fn emit_inconsistent_tree_node_amount_dp( + tree_nodes: &[TreeNode], + tip_distribution_account: &Pubkey, + rpc_client: &SyncRpcClient, +) { + let actual_claims: u64 = tree_nodes.iter().map(|t| t.amount).sum(); + let tda = rpc_client.get_account(tip_distribution_account).unwrap(); + let min_rent = rpc_client + .get_minimum_balance_for_rent_exemption(tda.data.len()) + .unwrap(); + + let expected_claims = tda.lamports.checked_sub(min_rent).unwrap(); + if actual_claims == expected_claims { + return; + } + + if actual_claims > expected_claims { + datapoint_error!( + "tip-distributor", + ( + "actual_claims_exceeded", + format!("tip_distribution_account={tip_distribution_account},actual_claims={actual_claims}, expected_claims={expected_claims}"), + String + ), + ); + } else { + datapoint_warn!( + "tip-distributor", + ( + "actual_claims_below", + format!("tip_distribution_account={tip_distribution_account},actual_claims={actual_claims}, expected_claims={expected_claims}"), + String + ), + ); + } +} + +impl GeneratedMerkleTreeCollection { + pub fn new_from_stake_meta_collection( + stake_meta_coll: StakeMetaCollection, + maybe_rpc_client: Option, + ) -> Result { + let generated_merkle_trees = stake_meta_coll + .stake_metas + .into_iter() + .filter(|stake_meta| stake_meta.maybe_tip_distribution_meta.is_some()) + .filter_map(|stake_meta| { + let mut tree_nodes = match TreeNode::vec_from_stake_meta(&stake_meta) { + Err(e) => return Some(Err(e)), + Ok(maybe_tree_nodes) => maybe_tree_nodes, + }?; + + if let Some(rpc_client) = &maybe_rpc_client { + if let Some(tda) = stake_meta.maybe_tip_distribution_meta.as_ref() { + emit_inconsistent_tree_node_amount_dp( + &tree_nodes[..], + &tda.tip_distribution_pubkey, + rpc_client, + ); + } + } + + let hashed_nodes: Vec<[u8; 32]> = + tree_nodes.iter().map(|n| n.hash().to_bytes()).collect(); + + let tip_distribution_meta = stake_meta.maybe_tip_distribution_meta.unwrap(); + + let merkle_tree = MerkleTree::new(&hashed_nodes[..], true); + let max_num_nodes = tree_nodes.len() as u64; + + for (i, tree_node) in tree_nodes.iter_mut().enumerate() { + tree_node.proof = Some(get_proof(&merkle_tree, i)); + } + + Some(Ok(GeneratedMerkleTree { + max_num_nodes, + tip_distribution_account: tip_distribution_meta.tip_distribution_pubkey, + merkle_root_upload_authority: tip_distribution_meta + .merkle_root_upload_authority, + merkle_root: *merkle_tree.get_root().unwrap(), + tree_nodes, + max_total_claim: tip_distribution_meta.total_tips, + })) + }) + .collect::, MerkleRootGeneratorError>>()?; + + Ok(GeneratedMerkleTreeCollection { + generated_merkle_trees, + bank_hash: stake_meta_coll.bank_hash, + epoch: stake_meta_coll.epoch, + slot: stake_meta_coll.slot, + }) + } +} + +pub fn get_proof(merkle_tree: &MerkleTree, i: usize) -> Vec<[u8; 32]> { + let mut proof = Vec::new(); + let path = merkle_tree.find_path(i).expect("path to index"); + for branch in path.get_proof_entries() { + if let Some(hash) = branch.get_left_sibling() { + proof.push(hash.to_bytes()); + } else if let Some(hash) = branch.get_right_sibling() { + proof.push(hash.to_bytes()); + } else { + panic!("expected some hash at each level of the tree"); + } + } + proof +} + +fn derive_tip_payment_pubkeys(program_id: &Pubkey) -> TipPaymentPubkeys { + let config_pda = Pubkey::find_program_address(&[CONFIG_ACCOUNT_SEED], program_id).0; + let tip_pda_0 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_0], program_id).0; + let tip_pda_1 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_1], program_id).0; + let tip_pda_2 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_2], program_id).0; + let tip_pda_3 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_3], program_id).0; + let tip_pda_4 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_4], program_id).0; + let tip_pda_5 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_5], program_id).0; + let tip_pda_6 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_6], program_id).0; + let tip_pda_7 = Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_7], program_id).0; + + TipPaymentPubkeys { + config_pda, + tip_pdas: vec![ + tip_pda_0, tip_pda_1, tip_pda_2, tip_pda_3, tip_pda_4, tip_pda_5, tip_pda_6, tip_pda_7, + ], + } +} + +#[derive(Clone, Eq, Debug, Hash, PartialEq, Deserialize, Serialize)] +pub struct TreeNode { + /// The stake account entitled to redeem. + #[serde(with = "pubkey_string_conversion")] + pub claimant: Pubkey, + + /// Pubkey of the ClaimStatus PDA account, this account should be closed to reclaim rent. + #[serde(with = "pubkey_string_conversion")] + pub claim_status_pubkey: Pubkey, + + /// Bump of the ClaimStatus PDA account + pub claim_status_bump: u8, + + #[serde(with = "pubkey_string_conversion")] + pub staker_pubkey: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub withdrawer_pubkey: Pubkey, + + /// The amount this account is entitled to. + pub amount: u64, + + /// The proof associated with this TreeNode + pub proof: Option>, +} + +impl TreeNode { + fn vec_from_stake_meta( + stake_meta: &StakeMeta, + ) -> Result>, MerkleRootGeneratorError> { + if let Some(tip_distribution_meta) = stake_meta.maybe_tip_distribution_meta.as_ref() { + let validator_amount = (tip_distribution_meta.total_tips as u128) + .checked_mul(tip_distribution_meta.validator_fee_bps as u128) + .unwrap() + .checked_div(10_000) + .unwrap() as u64; + let (claim_status_pubkey, claim_status_bump) = Pubkey::find_program_address( + &[ + ClaimStatus::SEED, + &stake_meta.validator_vote_account.to_bytes(), + &tip_distribution_meta.tip_distribution_pubkey.to_bytes(), + ], + &JitoTipDistribution::id(), + ); + let mut tree_nodes = vec![TreeNode { + claimant: stake_meta.validator_vote_account, + claim_status_pubkey, + claim_status_bump, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: validator_amount, + proof: None, + }]; + + let remaining_total_rewards = tip_distribution_meta + .total_tips + .checked_sub(validator_amount) + .unwrap() as u128; + + let total_delegated = stake_meta.total_delegated as u128; + tree_nodes.extend( + stake_meta + .delegations + .iter() + .map(|delegation| { + let amount_delegated = delegation.lamports_delegated as u128; + let reward_amount = (amount_delegated.checked_mul(remaining_total_rewards)) + .unwrap() + .checked_div(total_delegated) + .unwrap(); + let (claim_status_pubkey, claim_status_bump) = Pubkey::find_program_address( + &[ + ClaimStatus::SEED, + &delegation.stake_account_pubkey.to_bytes(), + &tip_distribution_meta.tip_distribution_pubkey.to_bytes(), + ], + &JitoTipDistribution::id(), + ); + Ok(TreeNode { + claimant: delegation.stake_account_pubkey, + claim_status_pubkey, + claim_status_bump, + staker_pubkey: delegation.staker_pubkey, + withdrawer_pubkey: delegation.withdrawer_pubkey, + amount: reward_amount as u64, + proof: None, + }) + }) + .collect::, MerkleRootGeneratorError>>()?, + ); + + Ok(Some(tree_nodes)) + } else { + Ok(None) + } + } + + fn hash(&self) -> Hash { + let mut hasher = Hasher::default(); + hasher.hash(self.claimant.as_ref()); + hasher.hash(self.amount.to_le_bytes().as_ref()); + hasher.result() + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct StakeMetaCollection { + /// List of [StakeMeta]. + pub stake_metas: Vec, + + /// base58 encoded tip-distribution program id. + #[serde(with = "pubkey_string_conversion")] + pub tip_distribution_program_id: Pubkey, + + /// Base58 encoded bank hash this object was generated at. + pub bank_hash: String, + + /// Epoch for which this object was generated for. + pub epoch: Epoch, + + /// Slot at which this object was generated. + pub slot: Slot, +} + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] +pub struct StakeMeta { + #[serde(with = "pubkey_string_conversion")] + pub validator_vote_account: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub validator_node_pubkey: Pubkey, + + /// The validator's tip-distribution meta if it exists. + pub maybe_tip_distribution_meta: Option, + + /// Delegations to this validator. + pub delegations: Vec, + + /// The total amount of delegations to the validator. + pub total_delegated: u64, + + /// The validator's delegation commission rate as a percentage between 0-100. + pub commission: u8, +} + +impl Ord for StakeMeta { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.validator_vote_account + .cmp(&other.validator_vote_account) + } +} + +impl PartialOrd for StakeMeta { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] +pub struct TipDistributionMeta { + #[serde(with = "pubkey_string_conversion")] + pub merkle_root_upload_authority: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub tip_distribution_pubkey: Pubkey, + + /// The validator's total tips in the [TipDistributionAccount]. + pub total_tips: u64, + + /// The validator's cut of tips from [TipDistributionAccount], calculated from the on-chain + /// commission fee bps. + pub validator_fee_bps: u16, +} + +impl TipDistributionMeta { + fn from_tda_wrapper( + tda_wrapper: TipDistributionAccountWrapper, + // The amount that will be left remaining in the tda to maintain rent exemption status. + rent_exempt_amount: u64, + ) -> Result { + Ok(TipDistributionMeta { + tip_distribution_pubkey: tda_wrapper.tip_distribution_pubkey, + total_tips: tda_wrapper + .account_data + .lamports() + .checked_sub(rent_exempt_amount) + .ok_or(CheckedMathError)?, + validator_fee_bps: tda_wrapper + .tip_distribution_account + .validator_commission_bps, + merkle_root_upload_authority: tda_wrapper + .tip_distribution_account + .merkle_root_upload_authority, + }) + } +} + +#[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] +pub struct Delegation { + #[serde(with = "pubkey_string_conversion")] + pub stake_account_pubkey: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub staker_pubkey: Pubkey, + + #[serde(with = "pubkey_string_conversion")] + pub withdrawer_pubkey: Pubkey, + + /// Lamports delegated by the stake account + pub lamports_delegated: u64, +} + +impl Ord for Delegation { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + ( + self.stake_account_pubkey, + self.withdrawer_pubkey, + self.staker_pubkey, + self.lamports_delegated, + ) + .cmp(&( + other.stake_account_pubkey, + other.withdrawer_pubkey, + other.staker_pubkey, + other.lamports_delegated, + )) + } +} + +impl PartialOrd for Delegation { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Convenience wrapper around [TipDistributionAccount] +pub struct TipDistributionAccountWrapper { + pub tip_distribution_account: TipDistributionAccount, + pub account_data: AccountSharedData, + pub tip_distribution_pubkey: Pubkey, +} + +// TODO: move to program's sdk +pub fn derive_tip_distribution_account_address( + tip_distribution_program_id: &Pubkey, + vote_pubkey: &Pubkey, + epoch: Epoch, +) -> (Pubkey, u8) { + Pubkey::find_program_address( + &[ + TipDistributionAccount::SEED, + vote_pubkey.to_bytes().as_ref(), + epoch.to_le_bytes().as_ref(), + ], + tip_distribution_program_id, + ) +} + +pub const MAX_RETRIES: usize = 5; +pub const FAIL_DELAY: Duration = Duration::from_millis(100); + +pub async fn sign_and_send_transactions_with_retries( + signer: &Keypair, + rpc_client: &RpcClient, + max_concurrent_rpc_get_reqs: usize, + transactions: Vec, + txn_send_batch_size: usize, + max_loop_duration: Duration, +) -> (Vec, HashMap) { + let semaphore = Arc::new(Semaphore::new(max_concurrent_rpc_get_reqs)); + let mut errors = HashMap::default(); + let mut blockhash = rpc_client + .get_latest_blockhash() + .await + .expect("fetch latest blockhash"); + // track unsigned txns + let mut transactions_to_process = transactions + .into_iter() + .map(|txn| (txn.message_data(), txn)) + .collect::, Transaction>>(); + + let start = Instant::now(); + while start.elapsed() < max_loop_duration && !transactions_to_process.is_empty() { + // ensure we always have a recent blockhash + // blockhashes last max 150 blocks + // finalized commitment is ~32 slots behind tip + // assuming 0% skip rate (every slot has a block), we’d have roughly 120 slots + // or (120*0.4s) = 48s to land a tx before it expires + // if we’re refreshing every 30s, then any txs sent immediately before the refresh would likely expire + if start.elapsed() > Duration::from_secs(1) { + blockhash = rpc_client + .get_latest_blockhash() + .await + .expect("fetch latest blockhash"); + } + info!( + "Sending {txn_send_batch_size} of {} transactions to claim mev tips", + transactions_to_process.len() + ); + let send_futs = transactions_to_process + .iter() + .take(txn_send_batch_size) + .map(|(hash, txn)| { + let semaphore = semaphore.clone(); + async move { + let _permit = semaphore.acquire_owned().await.unwrap(); // wait until our turn + let (txn, res) = signed_send(signer, rpc_client, blockhash, txn.clone()).await; + (hash.clone(), txn, res) + } + }); + + let send_res = futures::future::join_all(send_futs).await; + let new_errors = send_res + .into_iter() + .filter_map(|(hash, txn, result)| match result { + Err(e) => Some((txn.signatures[0], e)), + Ok(..) => { + let _ = transactions_to_process.remove(&hash); + None + } + }) + .collect::>(); + + errors.extend(new_errors); + } + + (transactions_to_process.values().cloned().collect(), errors) +} + +pub async fn send_until_blockhash_expires( + rpc_client: &RpcClient, + transactions: Vec, + blockhash: Hash, + keypair: &Arc, +) -> solana_rpc_client_api::client_error::Result<()> { + let mut claim_transactions: HashMap = transactions + .into_iter() + .map(|mut tx| { + tx.sign(&[&keypair], blockhash); + (*tx.get_signature(), tx) + }) + .collect(); + + let txs_requesting_send = claim_transactions.len(); + + while rpc_client + .is_blockhash_valid(&blockhash, CommitmentConfig::processed()) + .await? + { + let mut check_signatures = HashSet::with_capacity(claim_transactions.len()); + let mut already_processed = HashSet::with_capacity(claim_transactions.len()); + let mut is_blockhash_not_found = false; + + for (signature, tx) in &claim_transactions { + match rpc_client + .send_transaction_with_config( + tx, + RpcSendTransactionConfig { + skip_preflight: false, + preflight_commitment: Some(CommitmentLevel::Confirmed), + max_retries: Some(2), + ..RpcSendTransactionConfig::default() + }, + ) + .await + { + Ok(_) => { + check_signatures.insert(*signature); + } + Err(e) => match e.get_transaction_error() { + Some(TransactionError::BlockhashNotFound) => { + is_blockhash_not_found = true; + break; + } + Some(TransactionError::AlreadyProcessed) => { + already_processed.insert(*tx.get_signature()); + } + Some(e) => { + warn!( + "TransactionError sending signature: {} error: {:?} tx: {:?}", + tx.get_signature(), + e, + tx + ); + } + None => { + warn!( + "Unknown error sending transaction signature: {} error: {:?}", + tx.get_signature(), + e + ); + } + }, + } + } + + sleep(Duration::from_secs(10)).await; + + let signatures: Vec = check_signatures.iter().cloned().collect(); + let statuses = get_batched_signatures_statuses(rpc_client, &signatures).await?; + + for (signature, maybe_status) in &statuses { + if let Some(_status) = maybe_status { + claim_transactions.remove(signature); + check_signatures.remove(signature); + } + } + + for signature in already_processed { + claim_transactions.remove(&signature); + } + + if claim_transactions.is_empty() || is_blockhash_not_found { + break; + } + } + + let num_landed = txs_requesting_send + .checked_sub(claim_transactions.len()) + .unwrap(); + info!("num_landed: {:?}", num_landed); + + Ok(()) +} + +pub async fn get_batched_signatures_statuses( + rpc_client: &RpcClient, + signatures: &[Signature], +) -> solana_rpc_client_api::client_error::Result)>> { + let mut signature_statuses = Vec::new(); + + for signatures_batch in signatures.chunks(100) { + // was using get_signature_statuses_with_history, but it blocks if the signatures don't exist + // bigtable calls to read signatures that don't exist block forever w/o --rpc-bigtable-timeout argument set + // get_signature_statuses looks in status_cache, which only has a 150 block history + // may have false negative, but for this workflow it doesn't matter + let statuses = rpc_client.get_signature_statuses(signatures_batch).await?; + signature_statuses.extend(signatures_batch.iter().cloned().zip(statuses.value)); + } + Ok(signature_statuses) +} + +/// Just in time sign and send transaction to RPC +async fn signed_send( + signer: &Keypair, + rpc_client: &RpcClient, + blockhash: Hash, + mut txn: Transaction, +) -> (Transaction, solana_rpc_client_api::client_error::Result<()>) { + txn.sign(&[signer], blockhash); // just in time signing + let res = match rpc_client.send_and_confirm_transaction(&txn).await { + Ok(_) => Ok(()), + Err(e) => { + match e.kind { + // Already claimed, skip. + ErrorKind::TransactionError(TransactionError::AlreadyProcessed) + | ErrorKind::TransactionError(TransactionError::InstructionError( + 0, + InstructionError::Custom(0), + )) + | ErrorKind::RpcError(RpcError::RpcResponseError { + data: + RpcResponseErrorData::SendTransactionPreflightFailure( + RpcSimulateTransactionResult { + err: + Some(TransactionError::InstructionError( + 0, + InstructionError::Custom(0), + )), + .. + }, + ), + .. + }) => Ok(()), + + // transaction got held up too long and blockhash expired. retry txn + ErrorKind::TransactionError(TransactionError::BlockhashNotFound) => Err(e), + + // unexpected error, warn and retry + _ => { + error!( + "Error sending transaction. Signature: {}, Error: {e:?}", + txn.signatures[0] + ); + Err(e) + } + } + } + }; + + (txn, res) +} + +async fn get_batched_accounts( + rpc_client: &RpcClient, + pubkeys: &[Pubkey], +) -> solana_rpc_client_api::client_error::Result>> { + let mut batched_accounts = HashMap::new(); + + for pubkeys_chunk in pubkeys.chunks(MAX_MULTIPLE_ACCOUNTS) { + let accounts = rpc_client.get_multiple_accounts(pubkeys_chunk).await?; + batched_accounts.extend(pubkeys_chunk.iter().cloned().zip(accounts)); + } + Ok(batched_accounts) +} + +/// Calculates the minimum balance needed to be rent-exempt +/// taken from: https://github.com/jito-foundation/jito-solana/blob/d1ba42180d0093dd59480a77132477323a8e3f88/sdk/program/src/rent.rs#L78 +pub fn minimum_balance(data_len: usize) -> u64 { + ((((ACCOUNT_STORAGE_OVERHEAD + .checked_add(data_len as u64) + .unwrap()) + .checked_mul(DEFAULT_LAMPORTS_PER_BYTE_YEAR)) + .unwrap() as f64) + * DEFAULT_EXEMPTION_THRESHOLD) as u64 +} + +mod pubkey_string_conversion { + use { + serde::{self, Deserialize, Deserializer, Serializer}, + solana_sdk::pubkey::Pubkey, + std::str::FromStr, + }; + + pub(crate) fn serialize(pubkey: &Pubkey, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&pubkey.to_string()) + } + + pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Pubkey::from_str(&s).map_err(serde::de::Error::custom) + } +} + +pub fn read_json_from_file(path: &PathBuf) -> serde_json::Result +where + T: DeserializeOwned, +{ + let file = File::open(path).unwrap(); + let reader = BufReader::new(file); + serde_json::from_reader(reader) +} + +#[cfg(test)] +mod tests { + use {super::*, jito_tip_distribution::merkle_proof}; + + #[test] + fn test_merkle_tree_verify() { + // Create the merkle tree and proofs + let tda = Pubkey::new_unique(); + let (acct_0, acct_1) = (Pubkey::new_unique(), Pubkey::new_unique()); + let claim_statuses = &[(acct_0, tda), (acct_1, tda)] + .iter() + .map(|(claimant, tda)| { + Pubkey::find_program_address( + &[ClaimStatus::SEED, &claimant.to_bytes(), &tda.to_bytes()], + &JitoTipDistribution::id(), + ) + }) + .collect::>(); + let tree_nodes = vec![ + TreeNode { + claimant: acct_0, + claim_status_pubkey: claim_statuses[0].0, + claim_status_bump: claim_statuses[0].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 151_507, + proof: None, + }, + TreeNode { + claimant: acct_1, + claim_status_pubkey: claim_statuses[1].0, + claim_status_bump: claim_statuses[1].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 176_624, + proof: None, + }, + ]; + + // First the nodes are hashed and merkle tree constructed + let hashed_nodes: Vec<[u8; 32]> = tree_nodes.iter().map(|n| n.hash().to_bytes()).collect(); + let mk = MerkleTree::new(&hashed_nodes[..], true); + let root = mk.get_root().expect("to have valid root").to_bytes(); + + // verify first node + let node = solana_program::hash::hashv(&[&[0u8], &hashed_nodes[0]]); + let proof = get_proof(&mk, 0); + assert!(merkle_proof::verify(proof, root, node.to_bytes())); + + // verify second node + let node = solana_program::hash::hashv(&[&[0u8], &hashed_nodes[1]]); + let proof = get_proof(&mk, 1); + assert!(merkle_proof::verify(proof, root, node.to_bytes())); + } + + #[test] + fn test_new_from_stake_meta_collection_happy_path() { + let merkle_root_upload_authority = Pubkey::new_unique(); + + let (tda_0, tda_1) = (Pubkey::new_unique(), Pubkey::new_unique()); + + let stake_account_0 = Pubkey::new_unique(); + let stake_account_1 = Pubkey::new_unique(); + let stake_account_2 = Pubkey::new_unique(); + let stake_account_3 = Pubkey::new_unique(); + + let staker_account_0 = Pubkey::new_unique(); + let staker_account_1 = Pubkey::new_unique(); + let staker_account_2 = Pubkey::new_unique(); + let staker_account_3 = Pubkey::new_unique(); + + let validator_vote_account_0 = Pubkey::new_unique(); + let validator_vote_account_1 = Pubkey::new_unique(); + + let validator_id_0 = Pubkey::new_unique(); + let validator_id_1 = Pubkey::new_unique(); + + let stake_meta_collection = StakeMetaCollection { + stake_metas: vec![ + StakeMeta { + validator_vote_account: validator_vote_account_0, + validator_node_pubkey: validator_id_0, + maybe_tip_distribution_meta: Some(TipDistributionMeta { + merkle_root_upload_authority, + tip_distribution_pubkey: tda_0, + total_tips: 1_900_122_111_000, + validator_fee_bps: 100, + }), + delegations: vec![ + Delegation { + stake_account_pubkey: stake_account_0, + staker_pubkey: staker_account_0, + withdrawer_pubkey: staker_account_0, + lamports_delegated: 123_999_123_555, + }, + Delegation { + stake_account_pubkey: stake_account_1, + staker_pubkey: staker_account_1, + withdrawer_pubkey: staker_account_1, + lamports_delegated: 144_555_444_556, + }, + ], + total_delegated: 1_555_123_000_333_454_000, + commission: 100, + }, + StakeMeta { + validator_vote_account: validator_vote_account_1, + validator_node_pubkey: validator_id_1, + maybe_tip_distribution_meta: Some(TipDistributionMeta { + merkle_root_upload_authority, + tip_distribution_pubkey: tda_1, + total_tips: 1_900_122_111_333, + validator_fee_bps: 200, + }), + delegations: vec![ + Delegation { + stake_account_pubkey: stake_account_2, + staker_pubkey: staker_account_2, + withdrawer_pubkey: staker_account_2, + lamports_delegated: 224_555_444, + }, + Delegation { + stake_account_pubkey: stake_account_3, + staker_pubkey: staker_account_3, + withdrawer_pubkey: staker_account_3, + lamports_delegated: 700_888_944_555, + }, + ], + total_delegated: 2_565_318_909_444_123, + commission: 10, + }, + ], + tip_distribution_program_id: Pubkey::new_unique(), + bank_hash: Hash::new_unique().to_string(), + epoch: 100, + slot: 2_000_000, + }; + + let merkle_tree_collection = GeneratedMerkleTreeCollection::new_from_stake_meta_collection( + stake_meta_collection.clone(), + None, + ) + .unwrap(); + + assert_eq!(stake_meta_collection.epoch, merkle_tree_collection.epoch); + assert_eq!( + stake_meta_collection.bank_hash, + merkle_tree_collection.bank_hash + ); + assert_eq!(stake_meta_collection.slot, merkle_tree_collection.slot); + assert_eq!( + stake_meta_collection.stake_metas.len(), + merkle_tree_collection.generated_merkle_trees.len() + ); + let claim_statuses = &[ + (validator_vote_account_0, tda_0), + (stake_account_0, tda_0), + (stake_account_1, tda_0), + (validator_vote_account_1, tda_1), + (stake_account_2, tda_1), + (stake_account_3, tda_1), + ] + .iter() + .map(|(claimant, tda)| { + Pubkey::find_program_address( + &[ClaimStatus::SEED, &claimant.to_bytes(), &tda.to_bytes()], + &JitoTipDistribution::id(), + ) + }) + .collect::>(); + let tree_nodes = vec![ + TreeNode { + claimant: validator_vote_account_0, + claim_status_pubkey: claim_statuses[0].0, + claim_status_bump: claim_statuses[0].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 19_001_221_110, + proof: None, + }, + TreeNode { + claimant: stake_account_0, + claim_status_pubkey: claim_statuses[1].0, + claim_status_bump: claim_statuses[1].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 149_992, + proof: None, + }, + TreeNode { + claimant: stake_account_1, + claim_status_pubkey: claim_statuses[2].0, + claim_status_bump: claim_statuses[2].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 174_858, + proof: None, + }, + ]; + let hashed_nodes: Vec<[u8; 32]> = tree_nodes.iter().map(|n| n.hash().to_bytes()).collect(); + let merkle_tree = MerkleTree::new(&hashed_nodes[..], true); + let gmt_0 = GeneratedMerkleTree { + tip_distribution_account: tda_0, + merkle_root_upload_authority, + merkle_root: *merkle_tree.get_root().unwrap(), + tree_nodes, + max_total_claim: stake_meta_collection.stake_metas[0] + .clone() + .maybe_tip_distribution_meta + .unwrap() + .total_tips, + max_num_nodes: 3, + }; + + let tree_nodes = vec![ + TreeNode { + claimant: validator_vote_account_1, + claim_status_pubkey: claim_statuses[3].0, + claim_status_bump: claim_statuses[3].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 38_002_442_226, + proof: None, + }, + TreeNode { + claimant: stake_account_2, + claim_status_pubkey: claim_statuses[4].0, + claim_status_bump: claim_statuses[4].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 163_000, + proof: None, + }, + TreeNode { + claimant: stake_account_3, + claim_status_pubkey: claim_statuses[5].0, + claim_status_bump: claim_statuses[5].1, + staker_pubkey: Pubkey::default(), + withdrawer_pubkey: Pubkey::default(), + amount: 508_762_900, + proof: None, + }, + ]; + let hashed_nodes: Vec<[u8; 32]> = tree_nodes.iter().map(|n| n.hash().to_bytes()).collect(); + let merkle_tree = MerkleTree::new(&hashed_nodes[..], true); + let gmt_1 = GeneratedMerkleTree { + tip_distribution_account: tda_1, + merkle_root_upload_authority, + merkle_root: *merkle_tree.get_root().unwrap(), + tree_nodes, + max_total_claim: stake_meta_collection.stake_metas[1] + .clone() + .maybe_tip_distribution_meta + .unwrap() + .total_tips, + max_num_nodes: 3, + }; + + let expected_generated_merkle_trees = vec![gmt_0, gmt_1]; + let actual_generated_merkle_trees = merkle_tree_collection.generated_merkle_trees; + + expected_generated_merkle_trees + .iter() + .for_each(|expected_gmt| { + let actual_gmt = actual_generated_merkle_trees + .iter() + .find(|gmt| { + gmt.tip_distribution_account == expected_gmt.tip_distribution_account + }) + .unwrap(); + + assert_eq!(expected_gmt.max_num_nodes, actual_gmt.max_num_nodes); + assert_eq!(expected_gmt.max_total_claim, actual_gmt.max_total_claim); + assert_eq!( + expected_gmt.tip_distribution_account, + actual_gmt.tip_distribution_account + ); + assert_eq!(expected_gmt.tree_nodes.len(), actual_gmt.tree_nodes.len()); + expected_gmt + .tree_nodes + .iter() + .for_each(|expected_tree_node| { + let actual_tree_node = actual_gmt + .tree_nodes + .iter() + .find(|tree_node| tree_node.claimant == expected_tree_node.claimant) + .unwrap(); + assert_eq!(expected_tree_node.amount, actual_tree_node.amount); + }); + assert_eq!(expected_gmt.merkle_root, actual_gmt.merkle_root); + }); + } +} diff --git a/tip-distributor/src/merkle_root_generator_workflow.rs b/tip-distributor/src/merkle_root_generator_workflow.rs new file mode 100644 index 0000000000..bee3da016b --- /dev/null +++ b/tip-distributor/src/merkle_root_generator_workflow.rs @@ -0,0 +1,54 @@ +use { + crate::{read_json_from_file, GeneratedMerkleTreeCollection, StakeMetaCollection}, + log::*, + solana_client::rpc_client::RpcClient, + std::{ + fmt::Debug, + fs::File, + io::{BufWriter, Write}, + path::PathBuf, + }, + thiserror::Error, +}; + +#[derive(Error, Debug)] +pub enum MerkleRootGeneratorError { + #[error(transparent)] + IoError(#[from] std::io::Error), + + #[error(transparent)] + RpcError(#[from] Box), + + #[error(transparent)] + SerdeJsonError(#[from] serde_json::Error), +} + +pub fn generate_merkle_root( + stake_meta_coll_path: &PathBuf, + out_path: &PathBuf, + rpc_url: &str, +) -> Result<(), MerkleRootGeneratorError> { + let stake_meta_coll: StakeMetaCollection = read_json_from_file(stake_meta_coll_path)?; + + let rpc_client = RpcClient::new(rpc_url); + let merkle_tree_coll = GeneratedMerkleTreeCollection::new_from_stake_meta_collection( + stake_meta_coll, + Some(rpc_client), + )?; + + write_to_json_file(&merkle_tree_coll, out_path)?; + Ok(()) +} + +fn write_to_json_file( + merkle_tree_coll: &GeneratedMerkleTreeCollection, + file_path: &PathBuf, +) -> Result<(), MerkleRootGeneratorError> { + let file = File::create(file_path)?; + let mut writer = BufWriter::new(file); + let json = serde_json::to_string_pretty(&merkle_tree_coll).unwrap(); + writer.write_all(json.as_bytes())?; + writer.flush()?; + + Ok(()) +} diff --git a/tip-distributor/src/merkle_root_upload_workflow.rs b/tip-distributor/src/merkle_root_upload_workflow.rs new file mode 100644 index 0000000000..e40465581f --- /dev/null +++ b/tip-distributor/src/merkle_root_upload_workflow.rs @@ -0,0 +1,138 @@ +use { + crate::{ + read_json_from_file, sign_and_send_transactions_with_retries, GeneratedMerkleTree, + GeneratedMerkleTreeCollection, + }, + anchor_lang::AccountDeserialize, + jito_tip_distribution::{ + sdk::instruction::{upload_merkle_root_ix, UploadMerkleRootAccounts, UploadMerkleRootArgs}, + state::{Config, TipDistributionAccount}, + }, + log::{error, info}, + solana_client::nonblocking::rpc_client::RpcClient, + solana_program::{ + fee_calculator::DEFAULT_TARGET_LAMPORTS_PER_SIGNATURE, native_token::LAMPORTS_PER_SOL, + }, + solana_sdk::{ + commitment_config::CommitmentConfig, + pubkey::Pubkey, + signature::{read_keypair_file, Signer}, + transaction::Transaction, + }, + std::{path::PathBuf, time::Duration}, + thiserror::Error, + tokio::runtime::Builder, +}; + +#[derive(Error, Debug)] +pub enum MerkleRootUploadError { + #[error(transparent)] + IoError(#[from] std::io::Error), + + #[error(transparent)] + JsonError(#[from] serde_json::Error), +} + +pub fn upload_merkle_root( + merkle_root_path: &PathBuf, + keypair_path: &PathBuf, + rpc_url: &str, + tip_distribution_program_id: &Pubkey, + max_concurrent_rpc_get_reqs: usize, + txn_send_batch_size: usize, +) -> Result<(), MerkleRootUploadError> { + const MAX_RETRY_DURATION: Duration = Duration::from_secs(600); + + let merkle_tree: GeneratedMerkleTreeCollection = + read_json_from_file(merkle_root_path).expect("read GeneratedMerkleTreeCollection"); + let keypair = read_keypair_file(keypair_path).expect("read keypair file"); + + let tip_distribution_config = + Pubkey::find_program_address(&[Config::SEED], tip_distribution_program_id).0; + + let runtime = Builder::new_multi_thread() + .worker_threads(16) + .enable_all() + .build() + .expect("build runtime"); + + runtime.block_on(async move { + let rpc_client = + RpcClient::new_with_commitment(rpc_url.to_string(), CommitmentConfig::confirmed()); + let trees: Vec = merkle_tree + .generated_merkle_trees + .into_iter() + .filter(|tree| tree.merkle_root_upload_authority == keypair.pubkey()) + .collect(); + + info!("num trees to upload: {:?}", trees.len()); + + // heuristic to make sure we have enough funds to cover execution, assumes all trees need updating + { + let initial_balance = rpc_client.get_balance(&keypair.pubkey()).await.expect("failed to get balance"); + let desired_balance = (trees.len() as u64).checked_mul(DEFAULT_TARGET_LAMPORTS_PER_SIGNATURE).unwrap(); + if initial_balance < desired_balance { + let sol_to_deposit = desired_balance.checked_sub(initial_balance).unwrap().checked_add(LAMPORTS_PER_SOL).unwrap().checked_sub(1).unwrap().checked_div(LAMPORTS_PER_SOL).unwrap(); // rounds up to nearest sol + panic!("Expected to have at least {} lamports in {}, current balance is {} lamports, deposit {} SOL to continue.", + desired_balance, &keypair.pubkey(), initial_balance, sol_to_deposit) + } + } + let mut trees_needing_update: Vec = vec![]; + for tree in trees { + let account = rpc_client + .get_account(&tree.tip_distribution_account) + .await + .expect("fetch expect"); + + let mut data = account.data.as_slice(); + let fetched_tip_distribution_account = + TipDistributionAccount::try_deserialize(&mut data) + .expect("failed to deserialize tip_distribution_account state"); + + let needs_upload = match fetched_tip_distribution_account.merkle_root { + Some(merkle_root) => { + merkle_root.total_funds_claimed == 0 + && merkle_root.root != tree.merkle_root.to_bytes() + } + None => true, + }; + + if needs_upload { + trees_needing_update.push(tree); + } + } + + info!("num trees need uploading: {:?}", trees_needing_update.len()); + + let transactions: Vec = trees_needing_update + .iter() + .map(|tree| { + let ix = upload_merkle_root_ix( + *tip_distribution_program_id, + UploadMerkleRootArgs { + root: tree.merkle_root.to_bytes(), + max_total_claim: tree.max_total_claim, + max_num_nodes: tree.max_num_nodes, + }, + UploadMerkleRootAccounts { + config: tip_distribution_config, + merkle_root_upload_authority: keypair.pubkey(), + tip_distribution_account: tree.tip_distribution_account, + }, + ); + Transaction::new_with_payer( + &[ix], + Some(&keypair.pubkey()), + ) + }) + .collect(); + + let (to_process, failed_transactions) = sign_and_send_transactions_with_retries( + &keypair, &rpc_client, max_concurrent_rpc_get_reqs, transactions, txn_send_batch_size, MAX_RETRY_DURATION).await; + if !to_process.is_empty() { + panic!("{} remaining mev claim transactions, {} failed requests.", to_process.len(), failed_transactions.len()); + } + }); + + Ok(()) +} diff --git a/tip-distributor/src/reclaim_rent_workflow.rs b/tip-distributor/src/reclaim_rent_workflow.rs new file mode 100644 index 0000000000..fc48c89d61 --- /dev/null +++ b/tip-distributor/src/reclaim_rent_workflow.rs @@ -0,0 +1,310 @@ +use { + crate::{ + claim_mev_workflow::ClaimMevError, get_batched_accounts, + reclaim_rent_workflow::ClaimMevError::AnchorError, send_until_blockhash_expires, + }, + anchor_lang::AccountDeserialize, + jito_tip_distribution::{ + sdk::{ + derive_config_account_address, + instruction::{ + close_claim_status_ix, close_tip_distribution_account_ix, CloseClaimStatusAccounts, + CloseClaimStatusArgs, CloseTipDistributionAccountArgs, + CloseTipDistributionAccounts, + }, + }, + state::{ClaimStatus, Config, TipDistributionAccount}, + }, + log::{info, warn}, + rand::{prelude::SliceRandom, thread_rng}, + solana_client::nonblocking::rpc_client::RpcClient, + solana_metrics::datapoint_info, + solana_program::{clock::Epoch, pubkey::Pubkey}, + solana_rpc_client_api::config::RpcSimulateTransactionConfig, + solana_sdk::{ + account::Account, + commitment_config::CommitmentConfig, + compute_budget::ComputeBudgetInstruction, + signature::{Keypair, Signer}, + transaction::Transaction, + }, + std::{ + sync::Arc, + time::{Duration, Instant}, + }, +}; + +/// Clear old ClaimStatus accounts +pub async fn reclaim_rent( + rpc_url: String, + tip_distribution_program_id: Pubkey, + signer: Arc, + max_loop_duration: Duration, + // Optionally reclaim TipDistributionAccount rents on behalf of validators. + should_reclaim_tdas: bool, + micro_lamports: u64, +) -> Result<(), ClaimMevError> { + let rpc_client = RpcClient::new_with_timeout_and_commitment( + rpc_url.clone(), + Duration::from_secs(300), + CommitmentConfig::processed(), + ); + + let start = Instant::now(); + + let accounts = rpc_client + .get_program_accounts(&tip_distribution_program_id) + .await?; + + let config_pubkey = derive_config_account_address(&tip_distribution_program_id).0; + let config_account = rpc_client.get_account(&config_pubkey).await?; + let config_account = + Config::try_deserialize(&mut config_account.data.as_slice()).map_err(AnchorError)?; + + let epoch = rpc_client.get_epoch_info().await?.epoch; + let mut claim_status_pubkeys_to_expire = + find_expired_claim_status_accounts(&accounts, epoch, signer.pubkey()); + let mut tda_pubkeys_to_expire = find_expired_tda_accounts(&accounts, epoch); + + while start.elapsed() <= max_loop_duration { + let mut transactions = build_close_claim_status_transactions( + &claim_status_pubkeys_to_expire, + tip_distribution_program_id, + config_pubkey, + micro_lamports, + signer.pubkey(), + ); + if should_reclaim_tdas { + transactions.extend(build_close_tda_transactions( + &tda_pubkeys_to_expire, + tip_distribution_program_id, + config_pubkey, + &config_account, + signer.pubkey(), + )); + } + + datapoint_info!( + "claim_mev_workflow-prepare_rent_reclaim_transactions", + ("transaction_count", transactions.len(), i64), + ); + + if transactions.is_empty() { + info!("Finished reclaim rent!"); + return Ok(()); + } + + transactions.shuffle(&mut thread_rng()); + let transactions: Vec<_> = transactions.into_iter().take(10_000).collect(); + let blockhash = rpc_client.get_latest_blockhash().await?; + send_until_blockhash_expires(&rpc_client, transactions, blockhash, &signer).await?; + + // can just refresh calling get_multiple_accounts since these operations should be subtractive and not additive + let claim_status_pubkeys: Vec<_> = claim_status_pubkeys_to_expire + .iter() + .map(|(pubkey, _)| *pubkey) + .collect(); + claim_status_pubkeys_to_expire = get_batched_accounts(&rpc_client, &claim_status_pubkeys) + .await? + .into_iter() + .filter_map(|(pubkey, account)| Some((pubkey, account?))) + .collect(); + + let tda_pubkeys: Vec<_> = tda_pubkeys_to_expire + .iter() + .map(|(pubkey, _)| *pubkey) + .collect(); + tda_pubkeys_to_expire = get_batched_accounts(&rpc_client, &tda_pubkeys) + .await? + .into_iter() + .filter_map(|(pubkey, account)| Some((pubkey, account?))) + .collect(); + } + + // one final refresh before double checking everything + let claim_status_pubkeys: Vec<_> = claim_status_pubkeys_to_expire + .iter() + .map(|(pubkey, _)| *pubkey) + .collect(); + claim_status_pubkeys_to_expire = get_batched_accounts(&rpc_client, &claim_status_pubkeys) + .await? + .into_iter() + .filter_map(|(pubkey, account)| Some((pubkey, account?))) + .collect(); + + let tda_pubkeys: Vec<_> = tda_pubkeys_to_expire + .iter() + .map(|(pubkey, _)| *pubkey) + .collect(); + tda_pubkeys_to_expire = get_batched_accounts(&rpc_client, &tda_pubkeys) + .await? + .into_iter() + .filter_map(|(pubkey, account)| Some((pubkey, account?))) + .collect(); + + let mut transactions = build_close_claim_status_transactions( + &claim_status_pubkeys_to_expire, + tip_distribution_program_id, + config_pubkey, + micro_lamports, + signer.pubkey(), + ); + if should_reclaim_tdas { + transactions.extend(build_close_tda_transactions( + &tda_pubkeys_to_expire, + tip_distribution_program_id, + config_pubkey, + &config_account, + signer.pubkey(), + )); + } + + if transactions.is_empty() { + return Ok(()); + } + + // if more transactions left, we'll simulate them all to make sure its not an uncaught error + let mut is_error = false; + let mut error_str = String::new(); + for tx in &transactions { + match rpc_client + .simulate_transaction_with_config( + tx, + RpcSimulateTransactionConfig { + sig_verify: false, + replace_recent_blockhash: true, + commitment: Some(CommitmentConfig::processed()), + ..RpcSimulateTransactionConfig::default() + }, + ) + .await + { + Ok(_) => {} + Err(e) => { + error_str = e.to_string(); + is_error = true; + + match e.get_transaction_error() { + None => { + break; + } + Some(e) => { + warn!("transaction error. tx: {:?} error: {:?}", tx, e); + break; + } + } + } + } + } + + if is_error { + Err(ClaimMevError::UncaughtError { e: error_str }) + } else { + Err(ClaimMevError::NotFinished { + transactions_left: transactions.len(), + }) + } +} + +fn find_expired_claim_status_accounts( + accounts: &[(Pubkey, Account)], + epoch: Epoch, + payer: Pubkey, +) -> Vec<(Pubkey, Account)> { + accounts + .iter() + .filter_map(|(pubkey, account)| { + let claim_status = ClaimStatus::try_deserialize(&mut account.data.as_slice()).ok()?; + if claim_status.claim_status_payer.eq(&payer) && epoch > claim_status.expires_at { + Some((*pubkey, account.clone())) + } else { + None + } + }) + .collect() +} + +fn find_expired_tda_accounts( + accounts: &[(Pubkey, Account)], + epoch: Epoch, +) -> Vec<(Pubkey, Account)> { + accounts + .iter() + .filter_map(|(pubkey, account)| { + let tda = TipDistributionAccount::try_deserialize(&mut account.data.as_slice()).ok()?; + if epoch > tda.expires_at { + Some((*pubkey, account.clone())) + } else { + None + } + }) + .collect() +} + +/// Assumes accounts is already pre-filtered with checks to ensure the account can be closed +fn build_close_claim_status_transactions( + accounts: &[(Pubkey, Account)], + tip_distribution_program_id: Pubkey, + config: Pubkey, + microlamports: u64, + payer: Pubkey, +) -> Vec { + accounts + .iter() + .map(|(claim_status_pubkey, account)| { + let claim_status = ClaimStatus::try_deserialize(&mut account.data.as_slice()).unwrap(); + close_claim_status_ix( + tip_distribution_program_id, + CloseClaimStatusArgs, + CloseClaimStatusAccounts { + config, + claim_status: *claim_status_pubkey, + claim_status_payer: claim_status.claim_status_payer, + }, + ) + }) + .collect::>() + .chunks(4) + .map(|close_claim_status_instructions| { + let mut instructions = vec![ComputeBudgetInstruction::set_compute_unit_price( + microlamports, + )]; + instructions.extend(close_claim_status_instructions.to_vec()); + Transaction::new_with_payer(&instructions, Some(&payer)) + }) + .collect() +} + +fn build_close_tda_transactions( + accounts: &[(Pubkey, Account)], + tip_distribution_program_id: Pubkey, + config_pubkey: Pubkey, + config: &Config, + payer: Pubkey, +) -> Vec { + let instructions: Vec<_> = accounts + .iter() + .map(|(pubkey, account)| { + let tda = + TipDistributionAccount::try_deserialize(&mut account.data.as_slice()).unwrap(); + close_tip_distribution_account_ix( + tip_distribution_program_id, + CloseTipDistributionAccountArgs { + _epoch: tda.epoch_created_at, + }, + CloseTipDistributionAccounts { + config: config_pubkey, + tip_distribution_account: *pubkey, + validator_vote_account: tda.validator_vote_account, + expired_funds_account: config.expired_funds_account, + signer: payer, + }, + ) + }) + .collect(); + + instructions + .chunks(4) + .map(|ix_chunk| Transaction::new_with_payer(ix_chunk, Some(&payer))) + .collect() +} diff --git a/tip-distributor/src/stake_meta_generator_workflow.rs b/tip-distributor/src/stake_meta_generator_workflow.rs new file mode 100644 index 0000000000..9616154c09 --- /dev/null +++ b/tip-distributor/src/stake_meta_generator_workflow.rs @@ -0,0 +1,973 @@ +use { + crate::{ + derive_tip_distribution_account_address, derive_tip_payment_pubkeys, Config, StakeMeta, + StakeMetaCollection, TipDistributionAccount, TipDistributionAccountWrapper, + TipDistributionMeta, + }, + anchor_lang::AccountDeserialize, + itertools::Itertools, + log::*, + solana_accounts_db::hardened_unpack::{ + open_genesis_config, OpenGenesisConfigError, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + }, + solana_client::client_error::ClientError, + solana_ledger::{ + bank_forks_utils, + bank_forks_utils::BankForksUtilsError, + blockstore::{Blockstore, BlockstoreError}, + blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions}, + blockstore_processor::{BlockstoreProcessorError, ProcessOptions}, + }, + solana_program::{stake_history::StakeHistory, sysvar}, + solana_runtime::{bank::Bank, snapshot_config::SnapshotConfig, stakes::StakeAccount}, + solana_sdk::{ + account::{from_account, ReadableAccount, WritableAccount}, + clock::Slot, + pubkey::Pubkey, + }, + solana_vote::vote_account::VoteAccount, + std::{ + collections::HashMap, + fmt::{Debug, Display, Formatter}, + fs::File, + io::{BufWriter, Write}, + mem::size_of, + path::{Path, PathBuf}, + sync::{atomic::AtomicBool, Arc}, + }, + thiserror::Error, +}; + +#[derive(Error, Debug)] +pub enum StakeMetaGeneratorError { + #[error(transparent)] + AnchorError(#[from] Box), + + #[error(transparent)] + BlockstoreError(#[from] BlockstoreError), + + #[error(transparent)] + BlockstoreProcessorError(#[from] BlockstoreProcessorError), + + #[error(transparent)] + IoError(#[from] std::io::Error), + + CheckedMathError, + + #[error(transparent)] + RpcError(#[from] ClientError), + + #[error(transparent)] + SerdeJsonError(#[from] serde_json::Error), + + SnapshotSlotNotFound, + + BankForksUtilsError(#[from] BankForksUtilsError), + + GenesisConfigError(#[from] OpenGenesisConfigError), +} + +impl Display for StakeMetaGeneratorError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Debug::fmt(&self, f) + } +} + +/// Runs the entire workflow of creating a bank from a snapshot to writing stake meta-data +/// to a JSON file. +pub fn generate_stake_meta( + ledger_path: &Path, + snapshot_slot: &Slot, + tip_distribution_program_id: &Pubkey, + out_path: &str, + tip_payment_program_id: &Pubkey, +) -> Result<(), StakeMetaGeneratorError> { + info!("Creating bank from ledger path..."); + let bank = create_bank_from_snapshot(ledger_path, snapshot_slot)?; + + info!("Generating stake_meta_collection object..."); + let stake_meta_coll = + generate_stake_meta_collection(&bank, tip_distribution_program_id, tip_payment_program_id)?; + + info!("Writing stake_meta_collection to JSON {}...", out_path); + write_to_json_file(&stake_meta_coll, out_path)?; + + Ok(()) +} + +fn create_bank_from_snapshot( + ledger_path: &Path, + snapshot_slot: &Slot, +) -> Result, StakeMetaGeneratorError> { + let genesis_config = open_genesis_config(ledger_path, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE)?; + let snapshot_config = SnapshotConfig { + full_snapshot_archive_interval_slots: Slot::MAX, + incremental_snapshot_archive_interval_slots: Slot::MAX, + full_snapshot_archives_dir: PathBuf::from(ledger_path), + incremental_snapshot_archives_dir: PathBuf::from(ledger_path), + bank_snapshots_dir: PathBuf::from(ledger_path), + ..SnapshotConfig::default() + }; + let blockstore = Blockstore::open_with_options( + ledger_path, + BlockstoreOptions { + access_type: AccessType::PrimaryForMaintenance, + recovery_mode: None, + enforce_ulimit_nofile: false, + column_options: LedgerColumnOptions::default(), + }, + )?; + let (bank_forks, _, _) = bank_forks_utils::load_bank_forks( + &genesis_config, + &blockstore, + vec![PathBuf::from(ledger_path).join(Path::new("stake-meta.accounts"))], + Some(&snapshot_config), + &ProcessOptions::default(), + None, + None, + None, + Arc::new(AtomicBool::new(false)), + false, + )?; + + let working_bank = bank_forks.read().unwrap().working_bank(); + assert_eq!( + working_bank.slot(), + *snapshot_slot, + "expected working bank slot {}, found {}", + snapshot_slot, + working_bank.slot() + ); + + Ok(working_bank) +} + +fn write_to_json_file( + stake_meta_coll: &StakeMetaCollection, + out_path: &str, +) -> Result<(), StakeMetaGeneratorError> { + let file = File::create(out_path)?; + let mut writer = BufWriter::new(file); + let json = serde_json::to_string_pretty(&stake_meta_coll).unwrap(); + writer.write_all(json.as_bytes())?; + writer.flush()?; + + Ok(()) +} + +/// Creates a collection of [StakeMeta]'s from the given bank. +pub fn generate_stake_meta_collection( + bank: &Arc, + tip_distribution_program_id: &Pubkey, + tip_payment_program_id: &Pubkey, +) -> Result { + assert!(bank.is_frozen()); + + let epoch_vote_accounts = bank.epoch_vote_accounts(bank.epoch()).unwrap_or_else(|| { + panic!( + "No epoch_vote_accounts found for slot {} at epoch {}", + bank.slot(), + bank.epoch() + ) + }); + + let l_stakes = bank.stakes_cache.stakes(); + let delegations = l_stakes.stake_delegations(); + + let voter_pubkey_to_delegations = group_delegations_by_voter_pubkey(delegations, bank); + + // the last leader in an epoch may not crank the tip program before the epoch is over, which + // would result in MEV rewards for epoch N not being cranked until epoch N + 1. This means that + // the account balance in the snapshot could be incorrect. + // We assume that the rewards sitting in the tip program PDAs are cranked out by the time all of + // the rewards are claimed. + let tip_accounts = derive_tip_payment_pubkeys(tip_payment_program_id); + let account = bank + .get_account(&tip_accounts.config_pda) + .expect("config pda exists"); + + let config = Config::try_deserialize(&mut account.data()).expect("deserializes configuration"); + + let bb_commission_pct: u64 = config.block_builder_commission_pct; + let tip_receiver: Pubkey = config.tip_receiver; + + // includes the block builder fee + let excess_tip_balances: u64 = tip_accounts + .tip_pdas + .iter() + .map(|pubkey| { + let tip_account = bank.get_account(pubkey).expect("tip account exists"); + tip_account + .lamports() + .checked_sub(bank.get_minimum_balance_for_rent_exemption(tip_account.data().len())) + .expect("tip balance underflow") + }) + .sum(); + // matches math in tip payment program + let block_builder_tips = excess_tip_balances + .checked_mul(bb_commission_pct) + .expect("block_builder_tips overflow") + .checked_div(100) + .expect("block_builder_tips division error"); + let tip_receiver_fee = excess_tip_balances + .checked_sub(block_builder_tips) + .expect("tip_receiver_fee doesnt underflow"); + + let vote_pk_and_maybe_tdas: Vec<( + (Pubkey, &VoteAccount), + Option, + )> = epoch_vote_accounts + .iter() + .map(|(vote_pubkey, (_total_stake, vote_account))| { + let tip_distribution_pubkey = derive_tip_distribution_account_address( + tip_distribution_program_id, + vote_pubkey, + bank.epoch(), + ) + .0; + let tda = if let Some(mut account_data) = bank.get_account(&tip_distribution_pubkey) { + // TDAs may be funded with lamports and therefore exist in the bank, but would fail the deserialization step + // if the buffer is yet to be allocated thru the init call to the program. + if let Ok(tip_distribution_account) = + TipDistributionAccount::try_deserialize(&mut account_data.data()) + { + // this snapshot might have tips that weren't claimed by the time the epoch is over + // assume that it will eventually be cranked and credit the excess to this account + if tip_distribution_pubkey == tip_receiver { + account_data.set_lamports( + account_data + .lamports() + .checked_add(tip_receiver_fee) + .expect("tip overflow"), + ); + } + Some(TipDistributionAccountWrapper { + tip_distribution_account, + account_data, + tip_distribution_pubkey, + }) + } else { + None + } + } else { + None + }; + Ok(((*vote_pubkey, vote_account), tda)) + }) + .collect::>()?; + + let mut stake_metas = vec![]; + for ((vote_pubkey, vote_account), maybe_tda) in vote_pk_and_maybe_tdas { + if let Some(mut delegations) = voter_pubkey_to_delegations.get(&vote_pubkey).cloned() { + let total_delegated = delegations.iter().fold(0u64, |sum, delegation| { + sum.checked_add(delegation.lamports_delegated).unwrap() + }); + + let maybe_tip_distribution_meta = if let Some(tda) = maybe_tda { + let actual_len = tda.account_data.data().len(); + let expected_len = 8_usize.saturating_add(size_of::()); + if actual_len != expected_len { + warn!("len mismatch actual={actual_len}, expected={expected_len}"); + } + let rent_exempt_amount = + bank.get_minimum_balance_for_rent_exemption(tda.account_data.data().len()); + + Some(TipDistributionMeta::from_tda_wrapper( + tda, + rent_exempt_amount, + )?) + } else { + None + }; + + let vote_state = vote_account.vote_state().unwrap(); + delegations.sort(); + stake_metas.push(StakeMeta { + maybe_tip_distribution_meta, + validator_node_pubkey: vote_state.node_pubkey, + validator_vote_account: vote_pubkey, + delegations, + total_delegated, + commission: vote_state.commission, + }); + } else { + warn!( + "voter_pubkey not found in voter_pubkey_to_delegations map [validator_vote_pubkey={}]", + vote_pubkey + ); + } + } + stake_metas.sort(); + + Ok(StakeMetaCollection { + stake_metas, + tip_distribution_program_id: *tip_distribution_program_id, + bank_hash: bank.hash().to_string(), + epoch: bank.epoch(), + slot: bank.slot(), + }) +} + +/// Given an [EpochStakes] object, return delegations grouped by voter_pubkey (validator delegated to). +fn group_delegations_by_voter_pubkey( + delegations: &im::HashMap, + bank: &Bank, +) -> HashMap> { + delegations + .into_iter() + .filter(|(_stake_pubkey, stake_account)| { + stake_account.delegation().stake( + bank.epoch(), + &from_account::( + &bank.get_account(&sysvar::stake_history::id()).unwrap(), + ) + .unwrap(), + bank.new_warmup_cooldown_rate_epoch(), + ) > 0 + }) + .into_group_map_by(|(_stake_pubkey, stake_account)| stake_account.delegation().voter_pubkey) + .into_iter() + .map(|(voter_pubkey, group)| { + ( + voter_pubkey, + group + .into_iter() + .map(|(stake_pubkey, stake_account)| crate::Delegation { + stake_account_pubkey: *stake_pubkey, + staker_pubkey: stake_account + .stake_state() + .authorized() + .map(|a| a.staker) + .unwrap_or_default(), + withdrawer_pubkey: stake_account + .stake_state() + .authorized() + .map(|a| a.withdrawer) + .unwrap_or_default(), + lamports_delegated: stake_account.delegation().stake, + }) + .collect::>(), + ) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use { + super::*, + crate::derive_tip_distribution_account_address, + anchor_lang::AccountSerialize, + jito_tip_distribution::state::TipDistributionAccount, + jito_tip_payment::{ + InitBumps, TipPaymentAccount, CONFIG_ACCOUNT_SEED, TIP_ACCOUNT_SEED_0, + TIP_ACCOUNT_SEED_1, TIP_ACCOUNT_SEED_2, TIP_ACCOUNT_SEED_3, TIP_ACCOUNT_SEED_4, + TIP_ACCOUNT_SEED_5, TIP_ACCOUNT_SEED_6, TIP_ACCOUNT_SEED_7, + }, + solana_runtime::genesis_utils::{ + create_genesis_config_with_vote_accounts, GenesisConfigInfo, ValidatorVoteKeypairs, + }, + solana_sdk::{ + self, + account::{from_account, AccountSharedData}, + message::Message, + signature::{Keypair, Signer}, + stake::{ + self, + state::{Authorized, Lockup}, + }, + stake_history::StakeHistory, + sysvar, + transaction::Transaction, + }, + solana_stake_program::stake_state, + }; + + #[test] + fn test_generate_stake_meta_collection_happy_path() { + /* 1. Create a Bank seeded with some validator stake accounts */ + let validator_keypairs_0 = ValidatorVoteKeypairs::new_rand(); + let validator_keypairs_1 = ValidatorVoteKeypairs::new_rand(); + let validator_keypairs_2 = ValidatorVoteKeypairs::new_rand(); + let validator_keypairs = vec![ + &validator_keypairs_0, + &validator_keypairs_1, + &validator_keypairs_2, + ]; + const INITIAL_VALIDATOR_STAKES: u64 = 10_000; + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config_with_vote_accounts( + 1_000_000_000, + &validator_keypairs, + vec![INITIAL_VALIDATOR_STAKES; 3], + ); + + let (mut bank, _bank_forks) = Bank::new_with_bank_forks_for_tests(&genesis_config); + + /* 2. Seed the Bank with [TipDistributionAccount]'s */ + let merkle_root_upload_authority = Pubkey::new_unique(); + let tip_distribution_program_id = Pubkey::new_unique(); + let tip_payment_program_id = Pubkey::new_unique(); + + let delegator_0 = Keypair::new(); + let delegator_1 = Keypair::new(); + let delegator_2 = Keypair::new(); + let delegator_3 = Keypair::new(); + let delegator_4 = Keypair::new(); + + let delegator_0_pk = delegator_0.pubkey(); + let delegator_1_pk = delegator_1.pubkey(); + let delegator_2_pk = delegator_2.pubkey(); + let delegator_3_pk = delegator_3.pubkey(); + let delegator_4_pk = delegator_4.pubkey(); + + let d_0_data = AccountSharedData::new( + 300_000_000_000_000 * 10, + 0, + &solana_sdk::system_program::id(), + ); + let d_1_data = AccountSharedData::new( + 100_000_203_000_000 * 10, + 0, + &solana_sdk::system_program::id(), + ); + let d_2_data = AccountSharedData::new( + 100_000_235_899_000 * 10, + 0, + &solana_sdk::system_program::id(), + ); + let d_3_data = AccountSharedData::new( + 200_000_000_000_000 * 10, + 0, + &solana_sdk::system_program::id(), + ); + let d_4_data = AccountSharedData::new( + 100_000_000_777_000 * 10, + 0, + &solana_sdk::system_program::id(), + ); + + bank.store_account(&delegator_0_pk, &d_0_data); + bank.store_account(&delegator_1_pk, &d_1_data); + bank.store_account(&delegator_2_pk, &d_2_data); + bank.store_account(&delegator_3_pk, &d_3_data); + bank.store_account(&delegator_4_pk, &d_4_data); + + /* 3. Delegate some stake to the initial set of validators */ + let mut validator_0_delegations = vec![crate::Delegation { + stake_account_pubkey: validator_keypairs_0.stake_keypair.pubkey(), + staker_pubkey: validator_keypairs_0.stake_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_0.stake_keypair.pubkey(), + lamports_delegated: INITIAL_VALIDATOR_STAKES, + }]; + let stake_account = delegate_stake_helper( + &bank, + &delegator_0, + &validator_keypairs_0.vote_keypair.pubkey(), + 30_000_000_000, + ); + validator_0_delegations.push(crate::Delegation { + stake_account_pubkey: stake_account, + staker_pubkey: delegator_0.pubkey(), + withdrawer_pubkey: delegator_0.pubkey(), + lamports_delegated: 30_000_000_000, + }); + let stake_account = delegate_stake_helper( + &bank, + &delegator_1, + &validator_keypairs_0.vote_keypair.pubkey(), + 3_000_000_000, + ); + validator_0_delegations.push(crate::Delegation { + stake_account_pubkey: stake_account, + staker_pubkey: delegator_1.pubkey(), + withdrawer_pubkey: delegator_1.pubkey(), + lamports_delegated: 3_000_000_000, + }); + let stake_account = delegate_stake_helper( + &bank, + &delegator_2, + &validator_keypairs_0.vote_keypair.pubkey(), + 33_000_000_000, + ); + validator_0_delegations.push(crate::Delegation { + stake_account_pubkey: stake_account, + staker_pubkey: delegator_2.pubkey(), + withdrawer_pubkey: delegator_2.pubkey(), + lamports_delegated: 33_000_000_000, + }); + + let mut validator_1_delegations = vec![crate::Delegation { + stake_account_pubkey: validator_keypairs_1.stake_keypair.pubkey(), + staker_pubkey: validator_keypairs_1.stake_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_1.stake_keypair.pubkey(), + lamports_delegated: INITIAL_VALIDATOR_STAKES, + }]; + let stake_account = delegate_stake_helper( + &bank, + &delegator_3, + &validator_keypairs_1.vote_keypair.pubkey(), + 4_222_364_000, + ); + validator_1_delegations.push(crate::Delegation { + stake_account_pubkey: stake_account, + staker_pubkey: delegator_3.pubkey(), + withdrawer_pubkey: delegator_3.pubkey(), + lamports_delegated: 4_222_364_000, + }); + let stake_account = delegate_stake_helper( + &bank, + &delegator_4, + &validator_keypairs_1.vote_keypair.pubkey(), + 6_000_000_527, + ); + validator_1_delegations.push(crate::Delegation { + stake_account_pubkey: stake_account, + staker_pubkey: delegator_4.pubkey(), + withdrawer_pubkey: delegator_4.pubkey(), + lamports_delegated: 6_000_000_527, + }); + + let mut validator_2_delegations = vec![crate::Delegation { + stake_account_pubkey: validator_keypairs_2.stake_keypair.pubkey(), + staker_pubkey: validator_keypairs_2.stake_keypair.pubkey(), + withdrawer_pubkey: validator_keypairs_2.stake_keypair.pubkey(), + lamports_delegated: INITIAL_VALIDATOR_STAKES, + }]; + let stake_account = delegate_stake_helper( + &bank, + &delegator_0, + &validator_keypairs_2.vote_keypair.pubkey(), + 1_300_123_156, + ); + validator_2_delegations.push(crate::Delegation { + stake_account_pubkey: stake_account, + staker_pubkey: delegator_0.pubkey(), + withdrawer_pubkey: delegator_0.pubkey(), + lamports_delegated: 1_300_123_156, + }); + let stake_account = delegate_stake_helper( + &bank, + &delegator_4, + &validator_keypairs_2.vote_keypair.pubkey(), + 1_610_565_420, + ); + validator_2_delegations.push(crate::Delegation { + stake_account_pubkey: stake_account, + staker_pubkey: delegator_4.pubkey(), + withdrawer_pubkey: delegator_4.pubkey(), + lamports_delegated: 1_610_565_420, + }); + + /* 4. Run assertions */ + fn warmed_up(bank: &Bank, stake_pubkeys: &[Pubkey]) -> bool { + for stake_pubkey in stake_pubkeys { + let stake = + stake_state::stake_from(&bank.get_account(stake_pubkey).unwrap()).unwrap(); + + if stake.delegation.stake + != stake.stake( + bank.epoch(), + &from_account::( + &bank.get_account(&sysvar::stake_history::id()).unwrap(), + ) + .unwrap(), + bank.new_warmup_cooldown_rate_epoch(), + ) + { + return false; + } + } + + true + } + fn next_epoch(bank: &Arc) -> Arc { + bank.squash(); + + Arc::new(Bank::new_from_parent( + bank.clone(), + &Pubkey::default(), + bank.get_slots_in_epoch(bank.epoch()) + bank.slot(), + )) + } + + let mut stake_pubkeys = validator_0_delegations + .iter() + .map(|v| v.stake_account_pubkey) + .collect::>(); + stake_pubkeys.extend( + validator_1_delegations + .iter() + .map(|v| v.stake_account_pubkey), + ); + stake_pubkeys.extend( + validator_2_delegations + .iter() + .map(|v| v.stake_account_pubkey), + ); + loop { + if warmed_up(&bank, &stake_pubkeys[..]) { + break; + } + + // Cycle thru banks until we're fully warmed up + bank = next_epoch(&bank); + } + + let tip_distribution_account_0 = derive_tip_distribution_account_address( + &tip_distribution_program_id, + &validator_keypairs_0.vote_keypair.pubkey(), + bank.epoch(), + ); + let tip_distribution_account_1 = derive_tip_distribution_account_address( + &tip_distribution_program_id, + &validator_keypairs_1.vote_keypair.pubkey(), + bank.epoch(), + ); + let tip_distribution_account_2 = derive_tip_distribution_account_address( + &tip_distribution_program_id, + &validator_keypairs_2.vote_keypair.pubkey(), + bank.epoch(), + ); + + let expires_at = bank.epoch() + 3; + + let tda_0 = TipDistributionAccount { + validator_vote_account: validator_keypairs_0.vote_keypair.pubkey(), + merkle_root_upload_authority, + merkle_root: None, + epoch_created_at: bank.epoch(), + validator_commission_bps: 50, + expires_at, + bump: tip_distribution_account_0.1, + }; + let tda_1 = TipDistributionAccount { + validator_vote_account: validator_keypairs_1.vote_keypair.pubkey(), + merkle_root_upload_authority, + merkle_root: None, + epoch_created_at: bank.epoch(), + validator_commission_bps: 500, + expires_at: 0, + bump: tip_distribution_account_1.1, + }; + let tda_2 = TipDistributionAccount { + validator_vote_account: validator_keypairs_2.vote_keypair.pubkey(), + merkle_root_upload_authority, + merkle_root: None, + epoch_created_at: bank.epoch(), + validator_commission_bps: 75, + expires_at: 0, + bump: tip_distribution_account_2.1, + }; + + let tip_distro_0_tips = 1_000_000 * 10; + let tip_distro_1_tips = 69_000_420 * 10; + let tip_distro_2_tips = 789_000_111 * 10; + + let tda_0_fields = (tip_distribution_account_0.0, tda_0.validator_commission_bps); + let data_0 = + tda_to_account_shared_data(&tip_distribution_program_id, tip_distro_0_tips, tda_0); + let tda_1_fields = (tip_distribution_account_1.0, tda_1.validator_commission_bps); + let data_1 = + tda_to_account_shared_data(&tip_distribution_program_id, tip_distro_1_tips, tda_1); + let tda_2_fields = (tip_distribution_account_2.0, tda_2.validator_commission_bps); + let data_2 = + tda_to_account_shared_data(&tip_distribution_program_id, tip_distro_2_tips, tda_2); + + let accounts_data = create_config_account_data(&tip_payment_program_id, &bank); + for (pubkey, data) in accounts_data { + bank.store_account(&pubkey, &data); + } + + bank.store_account(&tip_distribution_account_0.0, &data_0); + bank.store_account(&tip_distribution_account_1.0, &data_1); + bank.store_account(&tip_distribution_account_2.0, &data_2); + + bank.freeze(); + let stake_meta_collection = generate_stake_meta_collection( + &bank, + &tip_distribution_program_id, + &tip_payment_program_id, + ) + .unwrap(); + assert_eq!( + stake_meta_collection.tip_distribution_program_id, + tip_distribution_program_id + ); + assert_eq!(stake_meta_collection.slot, bank.slot()); + assert_eq!(stake_meta_collection.epoch, bank.epoch()); + + let mut expected_stake_metas = HashMap::new(); + expected_stake_metas.insert( + validator_keypairs_0.vote_keypair.pubkey(), + StakeMeta { + validator_vote_account: validator_keypairs_0.vote_keypair.pubkey(), + delegations: validator_0_delegations.clone(), + total_delegated: validator_0_delegations + .iter() + .fold(0u64, |sum, delegation| { + sum.checked_add(delegation.lamports_delegated).unwrap() + }), + maybe_tip_distribution_meta: Some(TipDistributionMeta { + merkle_root_upload_authority, + tip_distribution_pubkey: tda_0_fields.0, + total_tips: tip_distro_0_tips + .checked_sub( + bank.get_minimum_balance_for_rent_exemption( + TipDistributionAccount::SIZE, + ), + ) + .unwrap(), + validator_fee_bps: tda_0_fields.1, + }), + commission: 0, + validator_node_pubkey: validator_keypairs_0.node_keypair.pubkey(), + }, + ); + expected_stake_metas.insert( + validator_keypairs_1.vote_keypair.pubkey(), + StakeMeta { + validator_vote_account: validator_keypairs_1.vote_keypair.pubkey(), + delegations: validator_1_delegations.clone(), + total_delegated: validator_1_delegations + .iter() + .fold(0u64, |sum, delegation| { + sum.checked_add(delegation.lamports_delegated).unwrap() + }), + maybe_tip_distribution_meta: Some(TipDistributionMeta { + merkle_root_upload_authority, + tip_distribution_pubkey: tda_1_fields.0, + total_tips: tip_distro_1_tips + .checked_sub( + bank.get_minimum_balance_for_rent_exemption( + TipDistributionAccount::SIZE, + ), + ) + .unwrap(), + validator_fee_bps: tda_1_fields.1, + }), + commission: 0, + validator_node_pubkey: validator_keypairs_1.node_keypair.pubkey(), + }, + ); + expected_stake_metas.insert( + validator_keypairs_2.vote_keypair.pubkey(), + StakeMeta { + validator_vote_account: validator_keypairs_2.vote_keypair.pubkey(), + delegations: validator_2_delegations.clone(), + total_delegated: validator_2_delegations + .iter() + .fold(0u64, |sum, delegation| { + sum.checked_add(delegation.lamports_delegated).unwrap() + }), + maybe_tip_distribution_meta: Some(TipDistributionMeta { + merkle_root_upload_authority, + tip_distribution_pubkey: tda_2_fields.0, + total_tips: tip_distro_2_tips + .checked_sub( + bank.get_minimum_balance_for_rent_exemption( + TipDistributionAccount::SIZE, + ), + ) + .unwrap(), + validator_fee_bps: tda_2_fields.1, + }), + commission: 0, + validator_node_pubkey: validator_keypairs_2.node_keypair.pubkey(), + }, + ); + + println!( + "validator_0 [vote_account={}, stake_account={}]", + validator_keypairs_0.vote_keypair.pubkey(), + validator_keypairs_0.stake_keypair.pubkey() + ); + println!( + "validator_1 [vote_account={}, stake_account={}]", + validator_keypairs_1.vote_keypair.pubkey(), + validator_keypairs_1.stake_keypair.pubkey() + ); + println!( + "validator_2 [vote_account={}, stake_account={}]", + validator_keypairs_2.vote_keypair.pubkey(), + validator_keypairs_2.stake_keypair.pubkey(), + ); + + assert_eq!( + expected_stake_metas.len(), + stake_meta_collection.stake_metas.len() + ); + + for actual_stake_meta in stake_meta_collection.stake_metas { + let expected_stake_meta = expected_stake_metas + .get(&actual_stake_meta.validator_vote_account) + .unwrap(); + assert_eq!( + expected_stake_meta.maybe_tip_distribution_meta, + actual_stake_meta.maybe_tip_distribution_meta + ); + assert_eq!( + expected_stake_meta.total_delegated, + actual_stake_meta.total_delegated + ); + assert_eq!(expected_stake_meta.commission, actual_stake_meta.commission); + assert_eq!( + expected_stake_meta.validator_vote_account, + actual_stake_meta.validator_vote_account + ); + + assert_eq!( + expected_stake_meta.delegations.len(), + actual_stake_meta.delegations.len() + ); + + for expected_delegation in &expected_stake_meta.delegations { + let actual_delegation = actual_stake_meta + .delegations + .iter() + .find(|d| d.stake_account_pubkey == expected_delegation.stake_account_pubkey) + .unwrap(); + + assert_eq!(expected_delegation, actual_delegation); + } + } + } + + /// Helper function that sends a delegate stake instruction to the bank. + /// Returns the created stake account pubkey. + fn delegate_stake_helper( + bank: &Bank, + from_keypair: &Keypair, + vote_account: &Pubkey, + delegation_amount: u64, + ) -> Pubkey { + let minimum_delegation = solana_stake_program::get_minimum_delegation(&bank.feature_set); + assert!( + delegation_amount >= minimum_delegation, + "{}", + format!( + "received delegation_amount {}, must be at least {}", + delegation_amount, minimum_delegation + ) + ); + if let Some(from_account) = bank.get_account(&from_keypair.pubkey()) { + assert_eq!(from_account.owner(), &solana_sdk::system_program::id()); + } else { + panic!("from_account DNE"); + } + assert!(bank.get_account(vote_account).is_some()); + + let stake_keypair = Keypair::new(); + let instructions = stake::instruction::create_account_and_delegate_stake( + &from_keypair.pubkey(), + &stake_keypair.pubkey(), + vote_account, + &Authorized::auto(&from_keypair.pubkey()), + &Lockup::default(), + delegation_amount, + ); + + let message = Message::new(&instructions[..], Some(&from_keypair.pubkey())); + let transaction = Transaction::new( + &[from_keypair, &stake_keypair], + message, + bank.last_blockhash(), + ); + + bank.process_transaction(&transaction) + .map_err(|e| { + eprintln!("Error delegating stake [error={}]", e); + e + }) + .unwrap(); + + stake_keypair.pubkey() + } + + fn tda_to_account_shared_data( + tip_distribution_program_id: &Pubkey, + lamports: u64, + tda: TipDistributionAccount, + ) -> AccountSharedData { + let mut account_data = AccountSharedData::new( + lamports, + TipDistributionAccount::SIZE, + tip_distribution_program_id, + ); + + let mut data: [u8; TipDistributionAccount::SIZE] = [0u8; TipDistributionAccount::SIZE]; + let mut cursor = std::io::Cursor::new(&mut data[..]); + tda.try_serialize(&mut cursor).unwrap(); + + account_data.set_data(data.to_vec()); + account_data + } + + fn create_config_account_data( + tip_payment_program_id: &Pubkey, + bank: &Bank, + ) -> Vec<(Pubkey, AccountSharedData)> { + let mut account_datas = vec![]; + + let config_pda = + Pubkey::find_program_address(&[CONFIG_ACCOUNT_SEED], tip_payment_program_id); + + let tip_accounts = [ + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_0], tip_payment_program_id), + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_1], tip_payment_program_id), + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_2], tip_payment_program_id), + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_3], tip_payment_program_id), + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_4], tip_payment_program_id), + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_5], tip_payment_program_id), + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_6], tip_payment_program_id), + Pubkey::find_program_address(&[TIP_ACCOUNT_SEED_7], tip_payment_program_id), + ]; + + let config = Config { + tip_receiver: Pubkey::new_unique(), + block_builder: Pubkey::new_unique(), + block_builder_commission_pct: 10, + bumps: InitBumps { + config: config_pda.1, + tip_payment_account_0: tip_accounts[0].1, + tip_payment_account_1: tip_accounts[1].1, + tip_payment_account_2: tip_accounts[2].1, + tip_payment_account_3: tip_accounts[3].1, + tip_payment_account_4: tip_accounts[4].1, + tip_payment_account_5: tip_accounts[5].1, + tip_payment_account_6: tip_accounts[6].1, + tip_payment_account_7: tip_accounts[7].1, + }, + }; + + let mut config_account_data = AccountSharedData::new( + bank.get_minimum_balance_for_rent_exemption(Config::SIZE), + Config::SIZE, + tip_payment_program_id, + ); + + let mut config_data: [u8; Config::SIZE] = [0u8; Config::SIZE]; + let mut config_cursor = std::io::Cursor::new(&mut config_data[..]); + config.try_serialize(&mut config_cursor).unwrap(); + config_account_data.set_data(config_data.to_vec()); + account_datas.push((config_pda.0, config_account_data)); + + account_datas.extend(tip_accounts.into_iter().map(|(pubkey, _)| { + let mut tip_account_data = AccountSharedData::new( + bank.get_minimum_balance_for_rent_exemption(TipPaymentAccount::SIZE), + TipPaymentAccount::SIZE, + tip_payment_program_id, + ); + + let mut data: [u8; TipPaymentAccount::SIZE] = [0u8; TipPaymentAccount::SIZE]; + let mut cursor = std::io::Cursor::new(&mut data[..]); + TipPaymentAccount::default() + .try_serialize(&mut cursor) + .unwrap(); + tip_account_data.set_data(data.to_vec()); + + (pubkey, tip_account_data) + })); + + account_datas + } +} diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 6a5eb5fb8d..67a8a4ba76 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -27,7 +27,7 @@ use { }, transaction_context::TransactionReturnData, }, - std::fmt, + std::{collections::HashMap, fmt}, thiserror::Error, }; @@ -301,6 +301,13 @@ impl From for UiInnerInstructions { } } +#[derive(Default)] +pub struct PreBalanceInfo { + pub native: Vec>, + pub token: Vec>, + pub mint_decimals: HashMap, +} + #[derive(Clone, Debug, PartialEq)] pub struct TransactionTokenBalance { pub account_index: u8, diff --git a/turbine/benches/cluster_info.rs b/turbine/benches/cluster_info.rs index 1f15137175..fffca1126f 100644 --- a/turbine/benches/cluster_info.rs +++ b/turbine/benches/cluster_info.rs @@ -76,6 +76,7 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) { &bank_forks, &SocketAddrSpace::Unspecified, &quic_endpoint_sender, + &None, ) .unwrap(); }); diff --git a/turbine/benches/retransmit_stage.rs b/turbine/benches/retransmit_stage.rs index c5490d5670..56c672c7c8 100644 --- a/turbine/benches/retransmit_stage.rs +++ b/turbine/benches/retransmit_stage.rs @@ -33,7 +33,7 @@ use { net::{Ipv4Addr, UdpSocket}, sync::{ atomic::{AtomicUsize, Ordering}, - Arc, + Arc, RwLock, }, thread::{sleep, Builder}, time::Duration, @@ -126,6 +126,7 @@ fn bench_retransmitter(bencher: &mut Bencher) { shreds_receiver, Arc::default(), // solana_rpc::max_slots::MaxSlots None, + Arc::new(RwLock::new(None)), ); let mut index = 0; diff --git a/turbine/src/broadcast_stage.rs b/turbine/src/broadcast_stage.rs index cce43f4fa5..29778e8962 100644 --- a/turbine/src/broadcast_stage.rs +++ b/turbine/src/broadcast_stage.rs @@ -118,6 +118,7 @@ impl BroadcastStageType { bank_forks: Arc>, shred_version: u16, quic_endpoint_sender: AsyncSender<(SocketAddr, Bytes)>, + shred_receiver_address: Arc>>, ) -> BroadcastStage { match self { BroadcastStageType::Standard => BroadcastStage::new( @@ -130,6 +131,7 @@ impl BroadcastStageType { bank_forks, quic_endpoint_sender, StandardBroadcastRun::new(shred_version), + shred_receiver_address, ), BroadcastStageType::FailEntryVerification => BroadcastStage::new( @@ -142,6 +144,7 @@ impl BroadcastStageType { bank_forks, quic_endpoint_sender, FailEntryVerificationBroadcastRun::new(shred_version), + Arc::new(RwLock::new(None)), ), BroadcastStageType::BroadcastFakeShreds => BroadcastStage::new( @@ -154,6 +157,7 @@ impl BroadcastStageType { bank_forks, quic_endpoint_sender, BroadcastFakeShredsRun::new(0, shred_version), + Arc::new(RwLock::new(None)), ), BroadcastStageType::BroadcastDuplicates(config) => BroadcastStage::new( @@ -166,6 +170,7 @@ impl BroadcastStageType { bank_forks, quic_endpoint_sender, BroadcastDuplicatesRun::new(shred_version, config.clone()), + Arc::new(RwLock::new(None)), ), } } @@ -187,6 +192,7 @@ trait BroadcastRun { sock: &UdpSocket, bank_forks: &RwLock, quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, + shred_receiver_address: &Arc>>, ) -> Result<()>; fn record(&mut self, receiver: &RecordReceiver, blockstore: &Blockstore) -> Result<()>; } @@ -282,6 +288,7 @@ impl BroadcastStage { bank_forks: Arc>, quic_endpoint_sender: AsyncSender<(SocketAddr, Bytes)>, mut broadcast_stage_run: impl BroadcastRun + Send + 'static + Clone, + shred_receiver_address: Arc>>, ) -> Self { let (socket_sender, socket_receiver) = unbounded(); let (blockstore_sender, blockstore_receiver) = unbounded(); @@ -313,6 +320,8 @@ impl BroadcastStage { let cluster_info = cluster_info.clone(); let bank_forks = bank_forks.clone(); let quic_endpoint_sender = quic_endpoint_sender.clone(); + let shred_receiver_address = shred_receiver_address.clone(); + let run_transmit = move || loop { let res = bs_transmit.transmit( &socket_receiver, @@ -320,6 +329,7 @@ impl BroadcastStage { &sock, &bank_forks, &quic_endpoint_sender, + &shred_receiver_address, ); let res = Self::handle_error(res, "solana-broadcaster-transmit"); if let Some(res) = res { @@ -429,6 +439,7 @@ fn update_peer_stats( /// Broadcasts shreds from the leader (i.e. this node) to the root of the /// turbine retransmit tree for each shred. +#[allow(clippy::too_many_arguments)] pub fn broadcast_shreds( s: &UdpSocket, shreds: &[Shred], @@ -439,6 +450,7 @@ pub fn broadcast_shreds( bank_forks: &RwLock, socket_addr_space: &SocketAddrSpace, quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, + shred_receiver_address: &Option, ) -> Result<()> { let mut result = Ok(()); let mut shred_select = Measure::start("shred_select"); @@ -454,15 +466,34 @@ pub fn broadcast_shreds( let cluster_nodes = cluster_nodes_cache.get(slot, &root_bank, &working_bank, cluster_info); update_peer_stats(&cluster_nodes, last_datapoint_submit); - shreds.filter_map(move |shred| { + shreds.flat_map(move |shred| { let key = shred.id(); let protocol = cluster_nodes::get_broadcast_protocol(&key); - cluster_nodes - .get_broadcast_peer(&key)? - .tvu(protocol) - .ok() - .filter(|addr| socket_addr_space.check(addr)) - .map(|addr| { + + let mut addrs = Vec::with_capacity(2); + if let Some(shred_receiver_address) = shred_receiver_address { + // Assuming always over UDP for shred_receiver_address + addrs.push((Protocol::UDP, *shred_receiver_address)); + } + if let Some(peer) = cluster_nodes.get_broadcast_peer(&key) { + match protocol { + Protocol::QUIC => { + if let Ok(tvu) = peer.tvu(Protocol::QUIC) { + addrs.push((Protocol::QUIC, tvu)); + } + } + Protocol::UDP => { + if let Ok(tvu) = peer.tvu(Protocol::UDP) { + addrs.push((Protocol::UDP, tvu)); + } + } + } + } + + addrs + .into_iter() + .filter(|(_, a)| socket_addr_space.check(a)) + .map(move |(protocol, addr)| { (match protocol { Protocol::QUIC => Either::Right, Protocol::UDP => Either::Left, @@ -697,6 +728,7 @@ pub mod test { bank_forks, quic_endpoint_sender, StandardBroadcastRun::new(0), + Arc::new(RwLock::new(None)), ); MockBroadcastStage { @@ -735,7 +767,10 @@ pub mod test { let ticks = create_ticks(max_tick_height - start_tick_height, 0, Hash::default()); for (i, tick) in ticks.into_iter().enumerate() { entry_sender - .send((bank.clone(), (tick, i as u64 + 1))) + .send(WorkingBankEntry { + bank: bank.clone(), + entries_ticks: vec![(tick, i as u64 + 1)], + }) .expect("Expect successful send to broadcast service"); } } diff --git a/turbine/src/broadcast_stage/broadcast_duplicates_run.rs b/turbine/src/broadcast_stage/broadcast_duplicates_run.rs index 3060fd27c8..64ef903151 100644 --- a/turbine/src/broadcast_stage/broadcast_duplicates_run.rs +++ b/turbine/src/broadcast_stage/broadcast_duplicates_run.rs @@ -297,6 +297,7 @@ impl BroadcastRun for BroadcastDuplicatesRun { sock: &UdpSocket, bank_forks: &RwLock, _quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, + _shred_receiver_addr: &Arc>>, ) -> Result<()> { let (shreds, _) = receiver.recv()?; if shreds.is_empty() { diff --git a/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs b/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs index b82ca324b6..649e8d0383 100644 --- a/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs +++ b/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs @@ -150,6 +150,7 @@ impl BroadcastRun for BroadcastFakeShredsRun { sock: &UdpSocket, _bank_forks: &RwLock, _quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, + _shred_receiver_addr: &Arc>>, ) -> Result<()> { for (data_shreds, batch_info) in receiver { let fake = batch_info.is_some(); diff --git a/turbine/src/broadcast_stage/broadcast_utils.rs b/turbine/src/broadcast_stage/broadcast_utils.rs index 9c4b48bd5c..c4795a840d 100644 --- a/turbine/src/broadcast_stage/broadcast_utils.rs +++ b/turbine/src/broadcast_stage/broadcast_utils.rs @@ -31,13 +31,23 @@ pub(super) fn recv_slot_entries(receiver: &Receiver) -> Result 32 * ShredData::capacity(/*merkle_proof_size*/ None).unwrap() as u64; let timer = Duration::new(1, 0); let recv_start = Instant::now(); - let (mut bank, (entry, mut last_tick_height)) = receiver.recv_timeout(timer)?; - let mut entries = vec![entry]; + + let WorkingBankEntry { + mut bank, + entries_ticks, + } = receiver.recv_timeout(timer)?; + let mut last_tick_height = entries_ticks.iter().last().unwrap().1; + let mut entries: Vec = entries_ticks.into_iter().map(|(e, _)| e).collect(); + assert!(last_tick_height <= bank.max_tick_height()); // Drain channel while last_tick_height != bank.max_tick_height() { - let Ok((try_bank, (entry, tick_height))) = receiver.try_recv() else { + let Ok(WorkingBankEntry { + bank: try_bank, + entries_ticks: new_entries_ticks, + }) = receiver.try_recv() + else { break; }; // If the bank changed, that implies the previous slot was interrupted and we do not have to @@ -47,8 +57,8 @@ pub(super) fn recv_slot_entries(receiver: &Receiver) -> Result entries.clear(); bank = try_bank; } - last_tick_height = tick_height; - entries.push(entry); + last_tick_height = new_entries_ticks.iter().last().unwrap().1; + entries.extend(new_entries_ticks.into_iter().map(|(entry, _)| entry)); assert!(last_tick_height <= bank.max_tick_height()); } @@ -59,8 +69,10 @@ pub(super) fn recv_slot_entries(receiver: &Receiver) -> Result while last_tick_height != bank.max_tick_height() && serialized_batch_byte_count < target_serialized_batch_byte_count { - let Ok((try_bank, (entry, tick_height))) = - receiver.recv_deadline(coalesce_start + ENTRY_COALESCE_DURATION) + let Ok(WorkingBankEntry { + bank: try_bank, + entries_ticks: new_entries_ticks, + }) = receiver.recv_deadline(coalesce_start + ENTRY_COALESCE_DURATION) else { break; }; @@ -73,10 +85,12 @@ pub(super) fn recv_slot_entries(receiver: &Receiver) -> Result bank = try_bank; coalesce_start = Instant::now(); } - last_tick_height = tick_height; - let entry_bytes = serialized_size(&entry)?; - serialized_batch_byte_count += entry_bytes; - entries.push(entry); + last_tick_height = new_entries_ticks.iter().last().unwrap().1; + + for (entry, _) in &new_entries_ticks { + serialized_batch_byte_count += serialized_size(entry)?; + } + entries.extend(new_entries_ticks.into_iter().map(|(entry, _)| entry)); assert!(last_tick_height <= bank.max_tick_height()); } let time_coalesced = coalesce_start.elapsed(); @@ -161,7 +175,11 @@ mod tests { .map(|i| { let entry = Entry::new(&last_hash, 1, vec![tx.clone()]); last_hash = entry.hash; - s.send((bank1.clone(), (entry.clone(), i))).unwrap(); + s.send(WorkingBankEntry { + bank: bank1.clone(), + entries_ticks: vec![(entry.clone(), i)], + }) + .unwrap(); entry }) .collect(); @@ -195,11 +213,18 @@ mod tests { last_hash = entry.hash; // Interrupt slot 1 right before the last tick if tick_height == expected_last_height { - s.send((bank2.clone(), (entry.clone(), tick_height))) - .unwrap(); + s.send(WorkingBankEntry { + bank: bank2.clone(), + entries_ticks: vec![(entry.clone(), tick_height)], + }) + .unwrap(); Some(entry) } else { - s.send((bank1.clone(), (entry, tick_height))).unwrap(); + s.send(WorkingBankEntry { + bank: bank1.clone(), + entries_ticks: vec![(entry, tick_height)], + }) + .unwrap(); None } }) diff --git a/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs b/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs index e9ed6a1a6e..c0fa09ce7d 100644 --- a/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs +++ b/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs @@ -3,7 +3,7 @@ use { crate::cluster_nodes::ClusterNodesCache, solana_ledger::shred::{ProcessShredsStats, ReedSolomonCache, Shredder}, solana_sdk::{hash::Hash, signature::Keypair}, - std::{thread::sleep, time::Duration}, + std::{net::SocketAddr, thread::sleep, time::Duration}, tokio::sync::mpsc::Sender as AsyncSender, }; @@ -180,6 +180,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun { sock: &UdpSocket, bank_forks: &RwLock, quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, + shred_receiver_address: &Arc>>, ) -> Result<()> { let (shreds, _) = receiver.recv()?; broadcast_shreds( @@ -192,6 +193,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun { bank_forks, cluster_info.socket_addr_space(), quic_endpoint_sender, + &shred_receiver_address.read().unwrap(), ) } fn record(&mut self, receiver: &RecordReceiver, blockstore: &Blockstore) -> Result<()> { diff --git a/turbine/src/broadcast_stage/standard_broadcast_run.rs b/turbine/src/broadcast_stage/standard_broadcast_run.rs index f108fc0822..39a466b8eb 100644 --- a/turbine/src/broadcast_stage/standard_broadcast_run.rs +++ b/turbine/src/broadcast_stage/standard_broadcast_run.rs @@ -17,7 +17,7 @@ use { signature::Keypair, timing::{duration_as_us, AtomicInterval}, }, - std::{sync::RwLock, time::Duration}, + std::{net::SocketAddr, sync::RwLock, time::Duration}, tokio::sync::mpsc::Sender as AsyncSender, }; @@ -179,10 +179,24 @@ impl StandardBroadcastRun { let (ssend, srecv) = unbounded(); self.process_receive_results(keypair, blockstore, &ssend, &bsend, receive_results)?; //data - let _ = self.transmit(&srecv, cluster_info, sock, bank_forks, quic_endpoint_sender); + let _ = self.transmit( + &srecv, + cluster_info, + sock, + bank_forks, + quic_endpoint_sender, + &Arc::new(RwLock::new(None)), + ); let _ = self.record(&brecv, blockstore); //coding - let _ = self.transmit(&srecv, cluster_info, sock, bank_forks, quic_endpoint_sender); + let _ = self.transmit( + &srecv, + cluster_info, + sock, + bank_forks, + quic_endpoint_sender, + &Arc::new(RwLock::new(None)), + ); let _ = self.record(&brecv, blockstore); Ok(()) } @@ -405,6 +419,7 @@ impl StandardBroadcastRun { broadcast_shred_batch_info: Option, bank_forks: &RwLock, quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, + shred_receiver_addr: &Option, ) -> Result<()> { trace!("Broadcasting {:?} shreds", shreds.len()); let mut transmit_stats = TransmitShredsStats::default(); @@ -421,6 +436,7 @@ impl StandardBroadcastRun { bank_forks, cluster_info.socket_addr_space(), quic_endpoint_sender, + shred_receiver_addr, )?; transmit_time.stop(); @@ -488,6 +504,7 @@ impl BroadcastRun for StandardBroadcastRun { sock: &UdpSocket, bank_forks: &RwLock, quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, + shred_receiver_address: &Arc>>, ) -> Result<()> { let (shreds, batch_info) = receiver.recv()?; self.broadcast( @@ -497,6 +514,7 @@ impl BroadcastRun for StandardBroadcastRun { batch_info, bank_forks, quic_endpoint_sender, + &shred_receiver_address.read().unwrap(), ) } fn record(&mut self, receiver: &RecordReceiver, blockstore: &Blockstore) -> Result<()> { diff --git a/turbine/src/retransmit_stage.rs b/turbine/src/retransmit_stage.rs index 32537db6b9..ec1616a9ab 100644 --- a/turbine/src/retransmit_stage.rs +++ b/turbine/src/retransmit_stage.rs @@ -184,6 +184,7 @@ fn retransmit( shred_deduper: &mut ShredDeduper<2>, max_slots: &MaxSlots, rpc_subscriptions: Option<&RpcSubscriptions>, + shred_receiver_address: &Arc>>, ) -> Result<(), RecvTimeoutError> { const RECV_TIMEOUT: Duration = Duration::from_secs(1); let mut shreds = shreds_receiver.recv_timeout(RECV_TIMEOUT)?; @@ -264,6 +265,7 @@ fn retransmit( &sockets[index % sockets.len()], quic_endpoint_sender, stats, + &shred_receiver_address.read().unwrap(), ) .map_err(|err| { stats.record_error(&err); @@ -289,6 +291,7 @@ fn retransmit( &sockets[index % sockets.len()], quic_endpoint_sender, stats, + &shred_receiver_address.read().unwrap(), ) .map_err(|err| { stats.record_error(&err); @@ -308,6 +311,7 @@ fn retransmit( Ok(()) } +#[allow(clippy::too_many_arguments)] fn retransmit_shred( key: &ShredId, shred: &[u8], @@ -318,15 +322,20 @@ fn retransmit_shred( socket: &UdpSocket, quic_endpoint_sender: &AsyncSender<(SocketAddr, Bytes)>, stats: &RetransmitStats, + shred_receiver_addr: &Option, ) -> Result<(/*root_distance:*/ usize, /*num_nodes:*/ usize), Error> { let mut compute_turbine_peers = Measure::start("turbine_start"); let data_plane_fanout = cluster_nodes::get_data_plane_fanout(key.slot(), root_bank); let (root_distance, addrs) = cluster_nodes.get_retransmit_addrs(slot_leader, key, data_plane_fanout)?; - let addrs: Vec<_> = addrs + let mut addrs: Vec<_> = addrs .into_iter() .filter(|addr| socket_addr_space.check(addr)) .collect(); + if let Some(addr) = shred_receiver_addr { + addrs.push(*addr); + } + compute_turbine_peers.stop(); stats .compute_turbine_peers_total @@ -383,6 +392,7 @@ pub fn retransmitter( shreds_receiver: Receiver>>, max_slots: Arc, rpc_subscriptions: Option>, + shred_receiver_addr: Arc>>, ) -> JoinHandle<()> { let cluster_nodes_cache = ClusterNodesCache::::new( CLUSTER_NODES_CACHE_NUM_EPOCH_CAP, @@ -414,6 +424,7 @@ pub fn retransmitter( &mut shred_deduper, &max_slots, rpc_subscriptions.as_deref(), + &shred_receiver_addr, ) { Ok(()) => (), Err(RecvTimeoutError::Timeout) => (), @@ -437,6 +448,7 @@ impl RetransmitStage { retransmit_receiver: Receiver>>, max_slots: Arc, rpc_subscriptions: Option>, + shred_receiver_addr: Arc>>, ) -> Self { let retransmit_thread_handle = retransmitter( retransmit_sockets, @@ -447,6 +459,7 @@ impl RetransmitStage { retransmit_receiver, max_slots, rpc_subscriptions, + shred_receiver_addr, ); Self { diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 39e5c4d1b1..0e2a028939 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -55,6 +55,7 @@ solana-rpc = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-runtime = { workspace = true } +solana-runtime-plugin = { workspace = true } solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } solana-storage-bigtable = { workspace = true } @@ -67,6 +68,7 @@ solana-vote-program = { workspace = true } symlink = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tonic = { workspace = true, features = ["tls", "tls-roots", "tls-webpki-roots"] } [dev-dependencies] assert_cmd = { workspace = true } diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index 99ef4b53a0..4cd7b44c93 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -12,6 +12,10 @@ use { solana_core::{ admin_rpc_post_init::AdminRpcRequestMetadataPostInit, consensus::{tower_storage::TowerStorage, Tower}, + proxy::{ + block_engine_stage::{BlockEngineConfig, BlockEngineStage}, + relayer_stage::{RelayerConfig, RelayerStage}, + }, repair::repair_service, validator::ValidatorStartProgress, }, @@ -30,6 +34,7 @@ use { fmt::{self, Display}, net::SocketAddr, path::{Path, PathBuf}, + str::FromStr, sync::{Arc, RwLock}, thread::{self, Builder}, time::{Duration, SystemTime}, @@ -243,6 +248,27 @@ pub trait AdminRpc { meta: Self::Metadata, public_tpu_forwards_addr: SocketAddr, ) -> Result<()>; + + #[rpc(meta, name = "setBlockEngineConfig")] + fn set_block_engine_config( + &self, + meta: Self::Metadata, + block_engine_url: String, + trust_packets: bool, + ) -> Result<()>; + + #[rpc(meta, name = "setRelayerConfig")] + fn set_relayer_config( + &self, + meta: Self::Metadata, + relayer_url: String, + trust_packets: bool, + expected_heartbeat_interval_ms: u64, + max_failed_heartbeats: u64, + ) -> Result<()>; + + #[rpc(meta, name = "setShredReceiverAddress")] + fn set_shred_receiver_address(&self, meta: Self::Metadata, addr: String) -> Result<()>; } pub struct AdminRpcImpl; @@ -441,6 +467,30 @@ impl AdminRpc for AdminRpcImpl { Ok(()) } + fn set_block_engine_config( + &self, + meta: Self::Metadata, + block_engine_url: String, + trust_packets: bool, + ) -> Result<()> { + debug!("set_block_engine_config request received"); + let config = BlockEngineConfig { + block_engine_url, + trust_packets, + }; + // Detailed log messages are printed inside validate function + if BlockEngineStage::is_valid_block_engine_config(&config) { + meta.with_post_init(|post_init| { + *post_init.block_engine_config.lock().unwrap() = config; + Ok(()) + }) + } else { + Err(jsonrpc_core::error::Error::invalid_params( + "failed to set block engine config. see logs for details.", + )) + } + } + fn set_identity( &self, meta: Self::Metadata, @@ -475,6 +525,55 @@ impl AdminRpc for AdminRpcImpl { AdminRpcImpl::set_identity_keypair(meta, identity_keypair, require_tower) } + fn set_relayer_config( + &self, + meta: Self::Metadata, + relayer_url: String, + trust_packets: bool, + expected_heartbeat_interval_ms: u64, + max_failed_heartbeats: u64, + ) -> Result<()> { + debug!("set_relayer_config request received"); + let expected_heartbeat_interval = Duration::from_millis(expected_heartbeat_interval_ms); + let oldest_allowed_heartbeat = + Duration::from_millis(max_failed_heartbeats * expected_heartbeat_interval_ms); + let config = RelayerConfig { + relayer_url, + expected_heartbeat_interval, + oldest_allowed_heartbeat, + trust_packets, + }; + // Detailed log messages are printed inside validate function + if RelayerStage::is_valid_relayer_config(&config) { + meta.with_post_init(|post_init| { + *post_init.relayer_config.lock().unwrap() = config; + Ok(()) + }) + } else { + Err(jsonrpc_core::error::Error::invalid_params( + "failed to set relayer config. see logs for details.", + )) + } + } + + fn set_shred_receiver_address(&self, meta: Self::Metadata, addr: String) -> Result<()> { + let shred_receiver_address = if addr.is_empty() { + None + } else { + Some(SocketAddr::from_str(&addr).map_err(|_| { + jsonrpc_core::error::Error::invalid_params(format!( + "invalid shred receiver address: {}", + addr + )) + })?) + }; + + meta.with_post_init(|post_init| { + *post_init.shred_receiver_address.write().unwrap() = shred_receiver_address; + Ok(()) + }) + } + fn set_staked_nodes_overrides(&self, meta: Self::Metadata, path: String) -> Result<()> { let loaded_config = load_staked_nodes_overrides(&path) .map_err(|err| { @@ -880,7 +979,10 @@ mod tests { solana_program::{program_option::COption, program_pack::Pack}, state::{Account as TokenAccount, AccountState as TokenAccountState, Mint}, }, - std::{collections::HashSet, sync::atomic::AtomicBool}, + std::{ + collections::HashSet, + sync::{atomic::AtomicBool, Mutex}, + }, }; #[derive(Default)] @@ -918,6 +1020,9 @@ mod tests { let vote_account = vote_keypair.pubkey(); let start_progress = Arc::new(RwLock::new(ValidatorStartProgress::default())); let repair_whitelist = Arc::new(RwLock::new(HashSet::new())); + let block_engine_config = Arc::new(Mutex::new(BlockEngineConfig::default())); + let relayer_config = Arc::new(Mutex::new(RelayerConfig::default())); + let shred_receiver_address = Arc::new(RwLock::new(None)); let meta = AdminRpcRequestMetadata { rpc_addr: None, start_time: SystemTime::now(), @@ -938,6 +1043,9 @@ mod tests { cluster_slots: Arc::new( solana_core::cluster_slots_service::cluster_slots::ClusterSlots::default(), ), + block_engine_config, + relayer_config, + shred_receiver_address, }))), staked_nodes_overrides: Arc::new(RwLock::new(HashMap::new())), rpc_to_plugin_manager_sender: None, diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index ef1ddf348b..dc195fae2a 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -816,12 +816,13 @@ fn get_highest_local_snapshot_hash( incremental_snapshot_archives_dir: impl AsRef, incremental_snapshot_fetch: bool, ) -> Option<(Slot, Hash)> { - snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir) + snapshot_utils::get_highest_full_snapshot_archive_info(full_snapshot_archives_dir, None) .and_then(|full_snapshot_info| { if incremental_snapshot_fetch { snapshot_utils::get_highest_incremental_snapshot_archive_info( incremental_snapshot_archives_dir, full_snapshot_info.slot(), + None, ) .map(|incremental_snapshot_info| { ( diff --git a/validator/src/cli.rs b/validator/src/cli.rs index 0f0b417039..5f396c3dc9 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -69,6 +69,10 @@ const MAX_SNAPSHOT_DOWNLOAD_ABORT: u32 = 5; // with less than 2 ticks per slot. const MINIMUM_TICKS_PER_SLOT: u64 = 2; +const DEFAULT_PREALLOCATED_BUNDLE_COST: &str = "3000000"; +const DEFAULT_RELAYER_EXPECTED_HEARTBEAT_INTERVAL_MS: &str = "500"; +const DEFAULT_RELAYER_MAX_FAILED_HEARTBEATS: &str = "3"; + pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { return App::new(crate_name!()) .about(crate_description!()) @@ -80,6 +84,87 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .long(SKIP_SEED_PHRASE_VALIDATION_ARG.long) .help(SKIP_SEED_PHRASE_VALIDATION_ARG.help), ) + .arg( + Arg::with_name("block_engine_url") + .long("block-engine-url") + .help("Block engine url. Set to empty string to disable block engine connection.") + .takes_value(true) + ) + .arg( + Arg::with_name("relayer_url") + .long("relayer-url") + .help("Relayer url. Set to empty string to disable relayer connection.") + .takes_value(true) + ) + .arg( + Arg::with_name("trust_relayer_packets") + .long("trust-relayer-packets") + .takes_value(false) + .help("Skip signature verification on relayer packets. Not recommended unless the relayer is trusted.") + ) + .arg( + Arg::with_name("relayer_expected_heartbeat_interval_ms") + .long("relayer-expected-heartbeat-interval-ms") + .takes_value(true) + .help("Interval at which the Relayer is expected to send heartbeat messages.") + .default_value(DEFAULT_RELAYER_EXPECTED_HEARTBEAT_INTERVAL_MS) + ) + .arg( + Arg::with_name("relayer_max_failed_heartbeats") + .long("relayer-max-failed-heartbeats") + .takes_value(true) + .help("Maximum number of heartbeats the Relayer can miss before falling back to the normal TPU pipeline.") + .default_value(DEFAULT_RELAYER_MAX_FAILED_HEARTBEATS) + ) + .arg( + Arg::with_name("trust_block_engine_packets") + .long("trust-block-engine-packets") + .takes_value(false) + .help("Skip signature verification on block engine packets. Not recommended unless the block engine is trusted.") + ) + .arg( + Arg::with_name("tip_payment_program_pubkey") + .long("tip-payment-program-pubkey") + .value_name("TIP_PAYMENT_PROGRAM_PUBKEY") + .takes_value(true) + .help("The public key of the tip-payment program") + ) + .arg( + Arg::with_name("tip_distribution_program_pubkey") + .long("tip-distribution-program-pubkey") + .value_name("TIP_DISTRIBUTION_PROGRAM_PUBKEY") + .takes_value(true) + .help("The public key of the tip-distribution program.") + ) + .arg( + Arg::with_name("merkle_root_upload_authority") + .long("merkle-root-upload-authority") + .value_name("MERKLE_ROOT_UPLOAD_AUTHORITY") + .takes_value(true) + .help("The public key of the authorized merkle-root uploader.") + ) + .arg( + Arg::with_name("commission_bps") + .long("commission-bps") + .value_name("COMMISSION_BPS") + .takes_value(true) + .help("The commission validator takes from tips expressed in basis points.") + ) + .arg( + Arg::with_name("preallocated_bundle_cost") + .long("preallocated-bundle-cost") + .value_name("PREALLOCATED_BUNDLE_COST") + .takes_value(true) + .default_value(DEFAULT_PREALLOCATED_BUNDLE_COST) + .help("Number of CUs to allocate for bundles at beginning of slot.") + ) + .arg( + Arg::with_name("shred_receiver_address") + .long("shred-receiver-address") + .value_name("SHRED_RECEIVER_ADDRESS") + .takes_value(true) + .help("Validator will forward all shreds to this address in addition to normal turbine operation. Set to empty string to disable.") + ) .arg( Arg::with_name("identity") .short("i") @@ -1198,7 +1283,14 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .value_name("BOOLEAN") .takes_value(true) .default_value("false") - .help("Еnable Geyser interface even if no Geyser configs are specified."), + .help("Еnable Geyser interface even if no Geyser configs are specified.")) +.arg( + Arg::with_name("runtime_plugin_config") + .long("runtime-plugin-config") + .value_name("FILE") + .takes_value(true) + .multiple(true) + .help("Specify the configuration file for a Runtime plugin.") ) .arg( Arg::with_name("snapshot_archive_format") @@ -1601,6 +1693,68 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .args(&thread_args(&default_args.thread_args)) .args(&get_deprecated_arguments()) .after_help("The default subcommand is run") + .subcommand( + SubCommand::with_name("set-block-engine-config") + .about("Set configuration for connection to a block engine") + .arg( + Arg::with_name("block_engine_url") + .long("block-engine-url") + .help("Block engine url. Set to empty string to disable block engine connection.") + .takes_value(true) + .required(true) + ) + .arg( + Arg::with_name("trust_block_engine_packets") + .long("trust-block-engine-packets") + .takes_value(false) + .help("Skip signature verification on block engine packets. Not recommended unless the block engine is trusted.") + ) + ) + .subcommand( + SubCommand::with_name("set-relayer-config") + .about("Set configuration for connection to a relayer") + .arg( + Arg::with_name("relayer_url") + .long("relayer-url") + .help("Relayer url. Set to empty string to disable relayer connection.") + .takes_value(true) + .required(true) + ) + .arg( + Arg::with_name("trust_relayer_packets") + .long("trust-relayer-packets") + .takes_value(false) + .help("Skip signature verification on relayer packets. Not recommended unless the relayer is trusted.") + ) + .arg( + Arg::with_name("relayer_expected_heartbeat_interval_ms") + .long("relayer-expected-heartbeat-interval-ms") + .takes_value(true) + .help("Interval at which the Relayer is expected to send heartbeat messages.") + .required(false) + .default_value(DEFAULT_RELAYER_EXPECTED_HEARTBEAT_INTERVAL_MS) + ) + .arg( + Arg::with_name("relayer_max_failed_heartbeats") + .long("relayer-max-failed-heartbeats") + .takes_value(true) + .help("Maximum number of heartbeats the Relayer can miss before falling back to the normal TPU pipeline.") + .required(false) + .default_value(DEFAULT_RELAYER_MAX_FAILED_HEARTBEATS) + ) + ) + .subcommand( + SubCommand::with_name("set-shred-receiver-address") + .about("Changes shred receiver address") + .arg( + Arg::with_name("shred_receiver_address") + .long("shred-receiver-address") + .value_name("SHRED_RECEIVER_ADDRESS") + .takes_value(true) + .help("Validator will forward all shreds to this address in addition to normal turbine operation. Set to empty string to disable.") + .required(true) + ) + ) .subcommand( SubCommand::with_name("exit") .about("Send an exit request to the validator") @@ -1777,6 +1931,48 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { ) .subcommand(SubCommand::with_name("monitor").about("Monitor the validator")) .subcommand(SubCommand::with_name("run").about("Run the validator")) + .subcommand( + SubCommand::with_name("runtime-plugin") + .about("Manage and view runtime plugins") + .setting(AppSettings::SubcommandRequiredElseHelp) + .setting(AppSettings::InferSubcommands) + .subcommand( + SubCommand::with_name("list") + .about("List all current running runtime plugins") + ) + .subcommand( + SubCommand::with_name("unload") + .about("Unload a particular runtime plugin. You must specify the runtime plugin name") + .arg( + Arg::with_name("name") + .required(true) + .takes_value(true) + ) + ) + .subcommand( + SubCommand::with_name("reload") + .about("Reload a particular runtime plugin. You must specify the runtime plugin name and the new config path") + .arg( + Arg::with_name("name") + .required(true) + .takes_value(true) + ) + .arg( + Arg::with_name("config") + .required(true) + .takes_value(true) + ) + ) + .subcommand( + SubCommand::with_name("load") + .about("Load a new gesyer plugin. You must specify the config path. Fails if overwriting (use reload)") + .arg( + Arg::with_name("config") + .required(true) + .takes_value(true) + ) + ) + ) .subcommand( SubCommand::with_name("plugin") .about("Manage and view geyser plugins") @@ -2776,6 +2972,14 @@ pub fn test_app<'a>(version: &'a str, default_args: &'a DefaultTestArgs) -> App< .multiple(true) .help("Specify the configuration file for the Geyser plugin."), ) + .arg( + Arg::with_name("runtime_plugin_config") + .long("runtime-plugin-config") + .value_name("FILE") + .takes_value(true) + .multiple(true) + .help("Specify the configuration file for a Runtime plugin."), + ) .arg( Arg::with_name("deactivate_feature") .long("deactivate-feature") diff --git a/validator/src/main.rs b/validator/src/main.rs index 45c35a43ef..4fb2a68c1e 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1,4 +1,5 @@ #![allow(clippy::arithmetic_side_effects)] + #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; use { @@ -30,7 +31,9 @@ use { solana_core::{ banking_trace::DISABLED_BAKING_TRACE_DIR, consensus::tower_storage, + proxy::{block_engine_stage::BlockEngineConfig, relayer_stage::RelayerConfig}, system_monitor_service::SystemMonitorService, + tip_manager::{TipDistributionAccountConfig, TipManagerConfig}, tpu::DEFAULT_TPU_COALESCE, validator::{ is_snapshot_config_valid, BlockProductionMethod, BlockVerificationMethod, Validator, @@ -63,6 +66,10 @@ use { snapshot_config::{SnapshotConfig, SnapshotUsage}, snapshot_utils::{self, ArchiveFormat, SnapshotVersion}, }, + solana_runtime_plugin::{ + runtime_plugin_admin_rpc_service, + runtime_plugin_admin_rpc_service::RuntimePluginAdminRpcRequestMetadata, + }, solana_sdk::{ clock::{Slot, DEFAULT_S_PER_SLOT}, commitment_config::CommitmentConfig, @@ -82,9 +89,10 @@ use { path::{Path, PathBuf}, process::exit, str::FromStr, - sync::{Arc, RwLock}, + sync::{atomic::AtomicBool, Arc, Mutex, RwLock}, time::{Duration, SystemTime}, }, + tokio::runtime::Runtime, }; #[cfg(not(target_env = "msvc"))] @@ -474,6 +482,60 @@ pub fn main() { let operation = match matches.subcommand() { ("", _) | ("run", _) => Operation::Run, + ("set-block-engine-config", Some(subcommand_matches)) => { + let block_engine_url = value_t_or_exit!(subcommand_matches, "block_engine_url", String); + let trust_packets = subcommand_matches.is_present("trust_block_engine_packets"); + let admin_client = admin_rpc_service::connect(&ledger_path); + admin_rpc_service::runtime() + .block_on(async move { + admin_client + .await? + .set_block_engine_config(block_engine_url, trust_packets) + .await + }) + .unwrap_or_else(|err| { + println!("set block engine config failed: {}", err); + exit(1); + }); + return; + } + ("set-relayer-config", Some(subcommand_matches)) => { + let relayer_url = value_t_or_exit!(subcommand_matches, "relayer_url", String); + let trust_packets = subcommand_matches.is_present("trust_relayer_packets"); + let expected_heartbeat_interval_ms: u64 = + value_of(subcommand_matches, "relayer_expected_heartbeat_interval_ms").unwrap(); + let max_failed_heartbeats: u64 = + value_of(subcommand_matches, "relayer_max_failed_heartbeats").unwrap(); + let admin_client = admin_rpc_service::connect(&ledger_path); + admin_rpc_service::runtime() + .block_on(async move { + admin_client + .await? + .set_relayer_config( + relayer_url, + trust_packets, + expected_heartbeat_interval_ms, + max_failed_heartbeats, + ) + .await + }) + .unwrap_or_else(|err| { + println!("set relayer config failed: {}", err); + exit(1); + }); + return; + } + ("set-shred-receiver-address", Some(subcommand_matches)) => { + let addr = value_t_or_exit!(subcommand_matches, "shred_receiver_address", String); + let admin_client = admin_rpc_service::connect(&ledger_path); + admin_rpc_service::runtime() + .block_on(async move { admin_client.await?.set_shred_receiver_address(addr).await }) + .unwrap_or_else(|err| { + println!("set shred receiver address failed: {}", err); + exit(1); + }); + return; + } ("authorized-voter", Some(authorized_voter_subcommand_matches)) => { match authorized_voter_subcommand_matches.subcommand() { ("add", Some(subcommand_matches)) => { @@ -625,6 +687,92 @@ pub fn main() { _ => unreachable!(), } } + ("runtime-plugin", Some(plugin_subcommand_matches)) => { + let runtime_plugin_rpc_client = runtime_plugin_admin_rpc_service::connect(&ledger_path); + let runtime = Runtime::new().unwrap(); + match plugin_subcommand_matches.subcommand() { + ("list", _) => { + let plugins = runtime + .block_on( + async move { runtime_plugin_rpc_client.await?.list_plugins().await }, + ) + .unwrap_or_else(|err| { + println!("Failed to list plugins: {err}"); + exit(1); + }); + if !plugins.is_empty() { + println!("Currently the following plugins are loaded:"); + for (plugin, i) in plugins.into_iter().zip(1..) { + println!(" {i}) {plugin}"); + } + } else { + println!("There are currently no plugins loaded"); + } + return; + } + ("unload", Some(subcommand_matches)) => { + if let Ok(name) = value_t!(subcommand_matches, "name", String) { + runtime + .block_on(async { + runtime_plugin_rpc_client + .await? + .unload_plugin(name.clone()) + .await + }) + .unwrap_or_else(|err| { + println!("Failed to unload plugin {name}: {err:?}"); + exit(1); + }); + println!("Successfully unloaded plugin: {name}"); + } + return; + } + ("load", Some(subcommand_matches)) => { + if let Ok(config) = value_t!(subcommand_matches, "config", String) { + let name = runtime + .block_on(async { + runtime_plugin_rpc_client + .await? + .load_plugin(config.clone()) + .await + }) + .unwrap_or_else(|err| { + println!("Failed to load plugin {config}: {err:?}"); + exit(1); + }); + println!("Successfully loaded plugin: {name}"); + } + return; + } + ("reload", Some(subcommand_matches)) => { + if let Ok(name) = value_t!(subcommand_matches, "name", String) { + if let Ok(config) = value_t!(subcommand_matches, "config", String) { + println!( + "This command does not work as intended on some systems.\ + To correctly reload an existing plugin make sure to:\ + 1. Rename the new plugin binary file.\ + 2. Unload the previous version.\ + 3. Load the new, renamed binary using the 'Load' command." + ); + runtime + .block_on(async { + runtime_plugin_rpc_client + .await? + .reload_plugin(name.clone(), config.clone()) + .await + }) + .unwrap_or_else(|err| { + println!("Failed to reload plugin {name}: {err:?}"); + exit(1); + }); + println!("Successfully reloaded plugin: {name}"); + } + } + return; + } + _ => unreachable!(), + } + } ("contact-info", Some(subcommand_matches)) => { let output_mode = subcommand_matches.value_of("output"); let admin_client = admin_rpc_service::connect(&ledger_path); @@ -1381,6 +1529,44 @@ pub fn main() { tvu_receive_threads, } = cli::thread_args::parse_num_threads_args(&matches); + let voting_disabled = matches.is_present("no_voting") || restricted_repair_only_mode; + let tip_manager_config = tip_manager_config_from_matches(&matches, voting_disabled); + + let block_engine_config = BlockEngineConfig { + block_engine_url: if matches.is_present("block_engine_url") { + value_of(&matches, "block_engine_url").expect("couldn't parse block_engine_url") + } else { + "".to_string() + }, + trust_packets: matches.is_present("trust_block_engine_packets"), + }; + + // Defaults are set in cli definition, safe to use unwrap() here + let expected_heartbeat_interval_ms: u64 = + value_of(&matches, "relayer_expected_heartbeat_interval_ms").unwrap(); + assert!( + expected_heartbeat_interval_ms > 0, + "relayer-max-failed-heartbeats must be greater than zero" + ); + let max_failed_heartbeats: u64 = value_of(&matches, "relayer_max_failed_heartbeats").unwrap(); + assert!( + max_failed_heartbeats > 0, + "relayer-max-failed-heartbeats must be greater than zero" + ); + + let relayer_config = RelayerConfig { + relayer_url: if matches.is_present("relayer_url") { + value_of(&matches, "relayer_url").expect("couldn't parse relayer_url") + } else { + "".to_string() + }, + expected_heartbeat_interval: Duration::from_millis(expected_heartbeat_interval_ms), + oldest_allowed_heartbeat: Duration::from_millis( + max_failed_heartbeats * expected_heartbeat_interval_ms, + ), + trust_packets: matches.is_present("trust_relayer_packets"), + }; + let mut validator_config = ValidatorConfig { require_tower: matches.is_present("require_tower"), tower_storage, @@ -1512,6 +1698,14 @@ pub fn main() { log_messages_bytes_limit: value_of(&matches, "log_messages_bytes_limit"), ..RuntimeConfig::default() }, + relayer_config: Arc::new(Mutex::new(relayer_config)), + block_engine_config: Arc::new(Mutex::new(block_engine_config)), + tip_manager_config, + shred_receiver_address: Arc::new(RwLock::new( + matches + .value_of("shred_receiver_address") + .map(|addr| SocketAddr::from_str(addr).expect("shred_receiver_address invalid")), + )), staked_nodes_overrides: staked_nodes_overrides.clone(), use_snapshot_archives_at_startup: value_t_or_exit!( matches, @@ -1523,6 +1717,8 @@ pub fn main() { replay_transactions_threads, delay_leader_block_for_pending_fork: matches .is_present("delay_leader_block_for_pending_fork"), + preallocated_bundle_cost: value_of(&matches, "preallocated_bundle_cost") + .expect("preallocated_bundle_cost set as default"), ..ValidatorConfig::default() }; @@ -1878,6 +2074,31 @@ pub fn main() { }, ); + let runtime_plugin_config_and_rpc_rx = { + let plugin_exit = Arc::new(AtomicBool::new(false)); + let (rpc_request_sender, rpc_request_receiver) = unbounded(); + solana_runtime_plugin::runtime_plugin_admin_rpc_service::run( + &ledger_path, + RuntimePluginAdminRpcRequestMetadata { + rpc_request_sender, + validator_exit: validator_config.validator_exit.clone(), + }, + plugin_exit, + ); + + if matches.is_present("runtime_plugin_config") { + ( + values_t_or_exit!(matches, "runtime_plugin_config", String) + .into_iter() + .map(PathBuf::from) + .collect(), + rpc_request_receiver, + ) + } else { + (vec![], rpc_request_receiver) + } + }; + let gossip_host: IpAddr = matches .value_of("gossip_host") .map(|gossip_host| { @@ -2055,6 +2276,7 @@ pub fn main() { tpu_enable_udp, tpu_max_connections_per_ipaddr_per_minute, admin_service_post_init, + Some(runtime_plugin_config_and_rpc_rx), ) .unwrap_or_else(|e| { error!("Failed to start validator: {:?}", e); @@ -2120,3 +2342,47 @@ fn process_account_indexes(matches: &ArgMatches) -> AccountSecondaryIndexes { indexes: account_indexes, } } + +fn tip_manager_config_from_matches( + matches: &ArgMatches, + voting_disabled: bool, +) -> TipManagerConfig { + TipManagerConfig { + tip_payment_program_id: pubkey_of(matches, "tip_payment_program_pubkey").unwrap_or_else( + || { + if !voting_disabled { + panic!("--tip-payment-program-pubkey argument required when validator is voting"); + } + Pubkey::new_unique() + }, + ), + tip_distribution_program_id: pubkey_of(matches, "tip_distribution_program_pubkey") + .unwrap_or_else(|| { + if !voting_disabled { + panic!("--tip-distribution-program-pubkey argument required when validator is voting"); + } + Pubkey::new_unique() + }), + tip_distribution_account_config: TipDistributionAccountConfig { + merkle_root_upload_authority: pubkey_of(matches, "merkle_root_upload_authority") + .unwrap_or_else(|| { + if !voting_disabled { + panic!("--merkle-root-upload-authority argument required when validator is voting"); + } + Pubkey::new_unique() + }), + vote_account: pubkey_of(matches, "vote_account").unwrap_or_else(|| { + if !voting_disabled { + panic!("--vote-account argument required when validator is voting"); + } + Pubkey::new_unique() + }), + commission_bps: value_t!(matches, "commission_bps", u16).unwrap_or_else(|_| { + if !voting_disabled { + panic!("--commission-bps argument required when validator is voting"); + } + 0 + }), + }, + } +} diff --git a/version/src/lib.rs b/version/src/lib.rs index a6f8e13b8a..b103433c26 100644 --- a/version/src/lib.rs +++ b/version/src/lib.rs @@ -80,7 +80,7 @@ impl Default for Version { commit: compute_commit(option_env!("CI_COMMIT")).unwrap_or_default(), feature_set, // Other client implementations need to modify this line. - client: u16::try_from(ClientId::Agave).unwrap(), + client: u16::try_from(ClientId::JitoLabs).unwrap(), } } } diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 07275e333c..e3a8265d85 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -452,7 +452,7 @@ pub(crate) fn generate_snapshot( // snapshot to use as base, so the logic is more complicated. For now we always generate // an incremental snapshot. let mut directory = &snapshot_config.full_snapshot_archives_dir; - let Some(full_snapshot_slot) = get_highest_full_snapshot_archive_slot(directory) else { + let Some(full_snapshot_slot) = get_highest_full_snapshot_archive_slot(directory, None) else { return Err(WenRestartError::MissingFullSnapshot( snapshot_config .full_snapshot_archives_dir @@ -472,7 +472,7 @@ pub(crate) fn generate_snapshot( check_slot_smaller_than_intended_snapshot_slot(full_snapshot_slot, new_root_slot, directory)?; directory = &snapshot_config.incremental_snapshot_archives_dir; if let Some(incremental_snapshot_slot) = - get_highest_incremental_snapshot_archive_slot(directory, full_snapshot_slot) + get_highest_incremental_snapshot_archive_slot(directory, full_snapshot_slot, None) { check_slot_smaller_than_intended_snapshot_slot( incremental_snapshot_slot,