test: add matrix dimension to run hive tests with experimental flag

.github/assets/hive/build_simulators.sh

@@ -35,4 +35,4 @@ wait
 # Make sure we don't rebuild images on the CI jobs
 git apply ../.github/assets/hive/no_sim_build.diff
 go build .
-mv ./hive ../hive_assets/
+mv ./hive ../hive_assets/

.github/assets/hive/expected_failures_experimental.yaml

@@ -0,0 +1,72 @@
+# https://github.com/paradigmxyz/reth/issues/7015
+# https://github.com/paradigmxyz/reth/issues/6332
+rpc-compat:
+  - debug_getRawBlock/get-invalid-number (reth)
+  - debug_getRawHeader/get-invalid-number (reth)
+  - debug_getRawReceipts/get-invalid-number (reth)
+  - debug_getRawTransaction/get-invalid-hash (reth)
+
+  - eth_call/call-callenv (reth)
+  - eth_createAccessList/create-al-contract-eip1559 (reth)
+  - eth_createAccessList/create-al-contract (reth)
+  - eth_feeHistory/fee-history (reth)
+  - eth_getStorageAt/get-storage-invalid-key-too-large (reth)
+  - eth_getStorageAt/get-storage-invalid-key (reth)
+  - eth_getTransactionReceipt/get-access-list (reth)
+  - eth_getTransactionReceipt/get-blob-tx (reth)
+  - eth_getTransactionReceipt/get-dynamic-fee (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8732
+engine-withdrawals:
+  - Withdrawals Fork On Genesis (Paris) (reth)
+  - Withdrawals Fork on Block 1 (Paris) (reth)
+  - Withdrawals Fork on Block 2 (Paris) (reth)
+  - Withdrawals Fork on Block 3 (Paris) (reth)
+  - Withdraw to a single account (Paris) (reth)
+  - Withdraw to two accounts (Paris) (reth)
+  - Withdraw many accounts (Paris) (reth)
+  - Withdraw zero amount (Paris) (reth)
+  - Empty Withdrawals (Paris) (reth)
+  - Corrupted Block Hash Payload (INVALID) (Paris) (reth)
+  - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
+  - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth)
+  - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
+  - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth)
+  - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+engine-api:
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Paris) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Paris) (reth)
+  - Re-org to Previously Validated Sidechain Payload (Paris) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P9 (Paris) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P10 (Paris) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P10 (Paris) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8305
+# https://github.com/paradigmxyz/reth/issues/6217
+# https://github.com/paradigmxyz/reth/issues/8306
+# https://github.com/paradigmxyz/reth/issues/7144
+engine-cancun:
+  - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, GasLimit, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
+  - Invalid NewPayload, BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, Blob Count on BlobGasUsed, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
+  - Re-org to Previously Validated Sidechain Payload (Cancun) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P9 (Cancun) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P10 (Cancun) (reth)
+  - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P10 (Cancun) (reth)
+
+# https://github.com/paradigmxyz/reth/issues/8579
+sync:
+  - sync reth -> reth

.github/workflows/hive.yml

@@ -23,7 +23,7 @@ jobs:
       group: Reth
     steps:
       - uses: actions/checkout@v4
-      - run: mkdir artifacts 
+      - run: mkdir artifacts
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
@@ -82,10 +82,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        engine: [regular, experimental]
         # ethereum/rpc to be deprecated:
         # https://github.com/ethereum/hive/pull/1117
-        sim: [smoke/genesis, smoke/network, ethereum/sync]
-        include:
+        scenario:
+          - sim: smoke/genesis
+          - sim: smoke/network
+          - sim: ethereum/sync
           - sim: devp2p
             limit: discv4
           - sim: devp2p
@@ -139,46 +142,46 @@ jobs:
               - eth_getTransactionReceipt
               - eth_sendRawTransaction
               - eth_syncing
-          # debug_ rpc methods
-          - sim: ethereum/rpc-compat
-            include: [debug_]
+              # debug_ rpc methods
+              - debug_
           # Pyspec cancun jobs
-          - sim: pyspec
-            include: [cancun/eip4844]
-          - sim: pyspec
-            include: [cancun/eip4788]
-          - sim: pyspec
-            include: [cancun/eip6780]
-          - sim: pyspec
-            include: [cancun/eip5656]
-          - sim: pyspec
-            include: [cancun/eip1153]
-          - sim: pyspec
-            include: [cancun/eip7516]
+          # TODO: uncomment when https://github.com/ethereum/hive/issues/1147 is fixed
+          #- sim: pyspec
+          #  include: [cancun/eip4844]
+          #- sim: pyspec
+          #  include: [cancun/eip4788]
+          #- sim: pyspec
+          #  include: [cancun/eip6780]
+          #- sim: pyspec
+          #  include: [cancun/eip5656]
+          #- sim: pyspec
+          #  include: [cancun/eip1153]
+          #- sim: pyspec
+          #  include: [cancun/eip7516]
           # Pyspec shanghai jobs
-          - sim: pyspec
-            include: [shanghai/eip3651]
-          - sim: pyspec
-            include: [shanghai/eip3855]
-          - sim: pyspec
-            include: [shanghai/eip3860]
-          - sim: pyspec
-            include: [shanghai/eip4895]
+          #- sim: pyspec
+          #  include: [shanghai/eip3651]
+          #- sim: pyspec
+          #  include: [shanghai/eip3855]
+          #- sim: pyspec
+          #  include: [shanghai/eip3860]
+          #- sim: pyspec
+          #  include: [shanghai/eip4895]
           # Pyspec merge and earlier jobs
-          - sim: pyspec
-            include: [merge/]
-          - sim: pyspec
-            include: [berlin/]
-          - sim: pyspec
-            include: [istanbul/]
-          - sim: pyspec
-            include: [homestead/]
-          - sim: pyspec
-            include: [frontier/]
-    needs: 
+          #- sim: pyspec
+          #  include: [merge/]
+          #- sim: pyspec
+          #  include: [berlin/]
+          #- sim: pyspec
+          #  include: [istanbul/]
+          #- sim: pyspec
+          #  include: [homestead/]
+          #- sim: pyspec
+          #  include: [frontier/]
+    needs:
       - prepare-reth
       - prepare-hive
-    name: run
+    name: run ${{ matrix.engine }} - ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
     runs-on:
       group: Reth
     permissions:
@@ -215,12 +218,31 @@ jobs:
           ref: master
           path: hivetests
 
-      - name: Run ${{ matrix.sim }} simulator
-        run: .github/assets/hive/run_simulator.sh "${{ matrix.sim }}$" "${{matrix.limit}}/${{join(matrix.include, '|')}}"
+      - name: Modify client for experimental engine
+        if: matrix.engine == 'experimental'
+        run: |
+          sed -ie 's/RUST_LOG=info $reth node $FLAGS/RUST_LOG=info $reth node --engine.experimental $FLAGS/' hivetests/clients/reth/reth.sh
+
+      - name: Run simulator
+        run: |
+          LIMIT="${{ matrix.scenario.limit }}"
+          TESTS="${{ join(matrix.scenario.include, '|') }}"
+          if [ -n "$LIMIT" ] && [ -n "$TESTS" ]; then
+            FILTER="$LIMIT/$TESTS"
+          elif [ -n "$LIMIT" ]; then
+            FILTER="$LIMIT"
+          elif [ -n "$TESTS" ]; then
+            FILTER="/$TESTS"
+          else
+            FILTER="/"
+          fi
+          echo "filter: $FILTER"
+          .github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
 
       - name: Parse hive output
         run: |
-          find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml
+          FAILURE_FILE="${{ matrix.engine == 'experimental' && '.github/assets/hive/expected_failures_experimental.yaml' || '.github/assets/hive/expected_failures.yaml' }}"
+          find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion $FAILURE_FILE
 
       - name: Print simulator output
         if: ${{ failure() }}