Skip to content

Commit

Permalink
Merge branch 'main' into reduce_dirty_space
Browse files Browse the repository at this point in the history
  • Loading branch information
dvovk committed Nov 7, 2024
2 parents 4a60bd4 + 5bdb221 commit 32510ea
Show file tree
Hide file tree
Showing 144 changed files with 3,485 additions and 2,243 deletions.
35 changes: 33 additions & 2 deletions .github/workflows/ci-cd-main-branch-docker-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ env:
CHECKOUT_REF: "main"
DOCKERHUB_REPOSITORY: "erigontech/erigon"
LABEL_DESCRIPTION: "[docker image built on a last commit id from the main branch] Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency frontier. Archive Node by default."
KEEP_IMAGES: 100

on:
push:
Expand Down Expand Up @@ -127,7 +128,7 @@ jobs:
--push \
--platform linux/amd64,linux/arm64 .
- name: export and print docker build tag
- name: export and print docker build tag, cleanup old docker images
id: built_tag_export
env:
BUILD_VERSION: "main-${{ steps.getCommitId.outputs.short_commit_id }}"
Expand All @@ -136,11 +137,41 @@ jobs:
echo The following docker images have been published:
echo "${{ env.DOCKERHUB_REPOSITORY }}:main-${{ env.BUILD_VERSION }}"
echo "${{ env.DOCKERHUB_REPOSITORY }}:main-latest"
echo
echo "Cleanup old docker images matching pattern tag ~= main-XXXXXXX"
curl_cmd="curl -s -H \"Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}\" "
dockerhub_url='https://hub.docker.com/v2/namespaces/erigontech/repositories/erigon'
my_list () {
# First page:
next_page="$dockerhub_url/tags?page=1&page_size=100"
while [ "$next_page" != "null" ]
do
# Print tags and push dates for tags matching "main-":
$curl_cmd $next_page | jq -r '.results|.[]|.name + " " + .tag_last_pushed' | grep 'main-'
next_page=`$curl_cmd $next_page | jq '.next' | sed -e 's/^\"//' -e 's/\"$//'`
done
}
my_list | tail -n+${{ env.KEEP_IMAGES }} | while read line; do
echo -n "Removing docker image/published - $line "
current_image=$(echo $line | sed -e 's/^\(main-.\{7\}\) .*/\1/')
output_code=$(curl --write-out %{http_code} --output curl-output.log \
-s -X DELETE -H "Accept: application/json" \
-H "Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}" \
https://hub.docker.com/v2/repositories/erigontech/erigon/tags/${current_image} )
if [ $output_code -ne 204 ]; then
echo "ERROR: failed to remove docker image erigon:${current_image}"
echo "ERROR: API response: $(cat curl-output.log)."
else
echo -n " - removed. "
fi
echo "Done."
done
run-kurtosis-assertoor:
needs: [define_matrix, Build]
uses: erigontech/erigon/.github/workflows/test-kurtosis-assertoor.yml@main
with:
checkout_ref: ${{ github.sha }}
os: ${{ needs.define_matrix.outputs.os }}
docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }}
docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }}
4 changes: 2 additions & 2 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ jobs:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: '1.22'
go-version: '1.23'

- name: Install golangci-lint
if: runner.os == 'Linux'
uses: golangci/golangci-lint-action@v6
with:
version: v1.59.1
version: v1.61.0
args: --help

- name: Lint
Expand Down
9 changes: 1 addition & 8 deletions .github/workflows/qa-constrained-tip-tracking.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,8 @@ name: QA - Constrained Tip tracking

on:
schedule:
- cron: '0 0 * * 0' # Run on Sunday at 00:00 AM UTC
- cron: '0 20 * * 0' # Run on Sunday at 08:00 PM UTC
workflow_dispatch: # Run manually
pull_request:
branches:
- qa_tests_constrained_tip_tracking
types:
- opened
- synchronize
- ready_for_review

jobs:
constrained-tip-tracking-test:
Expand Down
96 changes: 5 additions & 91 deletions .github/workflows/qa-rpc-integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ on:
jobs:
integration-test-suite:
runs-on: [ self-hosted, Erigon3 ]
timeout-minutes: 15
env:
ERIGON_REFERENCE_DATA_DIR: /opt/erigon-versions/reference-version/datadir
ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
Expand All @@ -31,7 +32,7 @@ jobs:
- name: Checkout RPC Tests Repository & Install Requirements
run: |
rm -rf ${{ runner.workspace }}/rpc-tests
git -c advice.detachedHead=false clone --depth 1 --branch v1.0.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
git -c advice.detachedHead=false clone --depth 1 --branch v1.7.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
cd ${{ runner.workspace }}/rpc-tests
pip3 install -r requirements.txt
Expand Down Expand Up @@ -83,97 +84,10 @@ jobs:
cd ${{ runner.workspace }}/rpc-tests/integration
rm -rf ./mainnet/results/
# Run RPC integration test runner via http
python3 ./run_tests.py -p 8545 --continue -f --json-diff -x \
debug_accountRange,debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber,debug_storageRangeAt,debug_traceBlockByHash,\
debug_traceCallMany/test_02.tar,debug_traceCallMany/test_04.tar,debug_traceCallMany/test_05.tar,debug_traceCallMany/test_06.tar,debug_traceCallMany/test_07.tar,debug_traceCallMany/test_09.json,debug_traceCallMany/test_10.tar,\
debug_traceBlockByNumber/test_05.tar,debug_traceBlockByNumber/test_08.tar,debug_traceBlockByNumber/test_09.tar,debug_traceBlockByNumber/test_10.tar,debug_traceBlockByNumber/test_11.tar,debug_traceBlockByNumber/test_12.tar,\
debug_traceTransaction,\
engine_exchangeCapabilities/test_1.json,\
engine_exchangeTransitionConfigurationV1/test_01.json,\
engine_getClientVersionV1/test_1.json,\
erigon_getLogsByHash/test_04.json,\
erigon_getHeaderByHash/test_02.json,\
erigon_getHeaderByHash/test_03.json,\
erigon_getHeaderByHash/test_04.json,\
erigon_getHeaderByHash/test_06.json,\
erigon_getHeaderByNumber/test_01.json,\
erigon_getHeaderByNumber/test_02.json,\
erigon_getHeaderByNumber/test_03.json,\
erigon_getHeaderByNumber/test_04.json,\
erigon_getHeaderByNumber/test_05.json,\
erigon_getHeaderByNumber/test_06.json,\
erigon_getHeaderByNumber/test_07.json,\
erigon_getHeaderByNumber/test_08.json,\
erigon_getLatestLogs/test_01.json,\
erigon_getLatestLogs/test_02.json,\
erigon_getLatestLogs/test_03.json,\
erigon_getLatestLogs/test_04.json,\
erigon_getLatestLogs/test_05.json,\
erigon_getLatestLogs/test_06.json,\
erigon_getLatestLogs/test_08.json,\
erigon_getLatestLogs/test_09.json,\
erigon_getLatestLogs/test_10.json,\
erigon_getLatestLogs/test_11.json,\
erigon_getLatestLogs/test_12.json,\
erigon_getBalanceChangesInBlock,\
eth_createAccessList/test_16.json,\
parity_getBlockReceipts/test_01.json,\
parity_getBlockReceipts/test_02.json,\
parity_getBlockReceipts/test_03.json,\
parity_getBlockReceipts/test_04.json,\
parity_getBlockReceipts/test_05.json,\
parity_getBlockReceipts/test_06.json,\
parity_getBlockReceipts/test_07.json,\
parity_getBlockReceipts/test_08.json,\
parity_getBlockReceipts/test_09.json,\
parity_getBlockReceipts/test_10.json,\
trace_filter/test_16.json,\
trace_rawTransaction/test_01.json,\
trace_rawTransaction/test_03.json,\
admin_nodeInfo/test_01.json,\
admin_peers/test_01.json,\
erigon_nodeInfo/test_1.json,\
eth_coinbase/test_01.json,\
eth_feeHistory/test_01.json,\
eth_feeHistory/test_02.json,\
eth_feeHistory/test_03.json,\
eth_feeHistory/test_04.json,\
eth_feeHistory/test_05.json,\
eth_feeHistory/test_06.json,\
eth_feeHistory/test_08.json,\
eth_feeHistory/test_09.json,\
eth_feeHistory/test_10.json,\
eth_feeHistory/test_11.json,\
eth_getBlockByHash/test_01.json,\
eth_getBlockByHash/test_02.json,\
eth_getBlockByHash/test_05.json,\
eth_getBlockByHash/test_06.json,\
eth_getBlockByHash/test_07.json,\
eth_getBlockByHash/test_08.json,\
eth_getBlockByNumber/test_01.json,\
eth_getBlockByNumber/test_02.json,\
eth_getBlockByNumber/test_04.json,\
eth_getBlockByNumber/test_05.json,\
eth_getBlockByNumber/test_06.json,\
eth_getBlockByNumber/test_07.json,\
eth_getBlockByNumber/test_08.json,\
eth_getBlockByNumber/test_12.json,\
eth_getBlockByNumber/test_13.json,\
eth_getTransactionByHash/test_02.json,\
eth_getWork/test_01.json,\
eth_mining/test_01.json,\
eth_protocolVersion/test_1.json,\
eth_submitHashrate/test_1.json,\
eth_submitWork/test_1.json,\
net_peerCount/test_1.json,\
net_version/test_1.json,\
txpool_content/test_01.json,\
txpool_status/test_1.json,\
web3_clientVersion/test_1.json,\
eth_estimateGas/test_14.json,\
trace_replayBlockTransactions/test_29.tar
chmod +x ${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests.sh
${{ runner.workspace }}/erigon/.github/workflows/scripts/run_rpc_tests.sh
# Capture test runner script exit status
test_exit_status=$?
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/qa-snap-download.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: QA - Snapshot Download

on:
schedule:
- cron: '0 22 * * 1-6' # Run every night at 22:00 (10:00 PM) UTC except Sunday
- cron: '0 20 * * 1-6' # Run every night at 20:00 (08:00 PM) UTC except Sunday
workflow_dispatch: # Run manually

jobs:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/qa-tip-tracking.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: QA - Tip tracking

on:
schedule:
- cron: '0 0 * * 1-6' # Run every night at 00:00 AM UTC except Sunday
- cron: '0 20 * * 1-6' # Run every night at 08:00 PM UTC except Sunday
workflow_dispatch: # Run manually

jobs:
Expand Down
74 changes: 74 additions & 0 deletions .github/workflows/scripts/run_rpc_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/bin/bash

set +e # Disable exit on error

# Array of disabled tests
disabled_tests=(
# Erigon2 and Erigon3 never supported this api methods
trace_rawTransaction
# remove these lines after https://github.com/erigontech/rpc-tests/pull/284 and https://github.com/erigontech/erigon/pull/12655
eth_feeHistory/test_12.json
eth_feeHistory/test_13.json
eth_feeHistory/test_15.json
eth_feeHistory/test_16.json
eth_feeHistory/test_17.json
eth_feeHistory/test_18.json
eth_feeHistory/test_19.json
eth_feeHistory/test_20.json
# total difficulty field was removed, then added back
# remove this line after https://github.com/erigontech/rpc-tests/pull/282
eth_getBlockByHash/test_10.json
eth_getBlockByNumber/test_12.json
# Erigon bugs
debug_accountRange,debug_storageRangeAt
# need update rpc-test - because Erigon is correct (@AskAlexSharov will do after https://github.com/erigontech/erigon/pull/12634)
# remove this line after https://github.com/erigontech/rpc-tests/pull/273
debug_getModifiedAccountsByHash,debug_getModifiedAccountsByNumber
# Erigon bug https://github.com/erigontech/erigon/issues/12603
erigon_getLatestLogs,erigon_getLogsByHash/test_04.json
# Erigon bug https://github.com/erigontech/erigon/issues/12637
debug_traceBlockByNumber/test_05.tar
debug_traceBlockByNumber/test_08.tar
debug_traceBlockByNumber/test_09.tar
debug_traceBlockByNumber/test_10.tar
debug_traceBlockByNumber/test_11.tar
debug_traceBlockByNumber/test_12.tar
# remove this line after https://github.com/erigontech/rpc-tests/pull/281
parity_getBlockReceipts
parity_listStorageKeys/test_12.json
# to investigate
debug_traceCallMany/test_02.tar
debug_traceCallMany/test_04.tar
debug_traceCallMany/test_05.tar
debug_traceCallMany/test_06.tar
debug_traceCallMany/test_07.tar
debug_traceCallMany/test_09.json
debug_traceCallMany/test_10.tar
engine_exchangeCapabilities/test_1.json
engine_exchangeTransitionConfigurationV1/test_01.json
engine_getClientVersionV1/test_1.json
erigon_getBalanceChangesInBlock
trace_replayBlockTransactions/test_29.tar
# do these perhaps require Erigon up?
admin_nodeInfo/test_01.json
admin_peers/test_01.json
erigon_nodeInfo/test_1.json
eth_coinbase/test_01.json
eth_createAccessList/test_16.json
eth_getTransactionByHash/test_02.json
eth_getWork/test_01.json
eth_mining/test_01.json
eth_protocolVersion/test_1.json
eth_submitHashrate/test_1.json
eth_submitWork/test_1.json
net_peerCount/test_1.json
net_version/test_1.json
txpool_status/test_1.json
web3_clientVersion/test_1.json)

# Transform the array into a comma-separated string
disabled_test_list=$(IFS=,; echo "${disabled_tests[*]}")

python3 ./run_tests.py -p 8545 --continue -f --json-diff -x "$disabled_test_list"

exit $?
81 changes: 81 additions & 0 deletions .github/workflows/test-hive-eest.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
name: Hive EEST tests

on:
push:
branches:
- release/*
- main
workflow_dispatch:

jobs:
test-hive-eest:
runs-on: ubuntu-latest
steps:
- name: Checkout Hive
uses: actions/checkout@v4
with:
repository: danceratopz/hive
ref: prague-devnet-4
path: hive
- name: Setup go env and cache
uses: actions/setup-go@v5
with:
go-version: '>=1.22'
go-version-file: 'hive/go.mod'

# Targetting the clients/erigon/Dockerfile.git in the Hive director -
# this builds the container from github and uses it for tests
- name: Get dependencies and build hive
run: |
cd hive
git status
go get . >> buildlogs.log
rm clients/erigon/Dockerfile
mv clients/erigon/Dockerfile.git clients/erigon/Dockerfile
branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/[&/\]/\\&/g')
echo Building Hive with Erigon branch - $branch_name
sed -i "s/^ARG github=ledgerwatch\/erigon$/ARG github=erigontech\/erigon/" clients/erigon/Dockerfile
sed -i "s/^ARG tag=main$/ARG tag=${branch_name}/" clients/erigon/Dockerfile
if [[ "$branch_name" != "main" ]]; then
sed -i "/$sync.parallel-state-flushing/d" clients/erigon/erigon.sh
fi
go build . >> buildlogs.log
# Depends on the last line of hive output that prints the number of suites, tests and failed
# Currently, we fail even if suites and tests are too few, indicating the tests did not run
# We also fail if more than half the tests fail
- name: Run hive tests and parse output
run: |
cd hive
run_suite() {
echo -e "\n\n============================================================"
echo "Running test: ${1}"
echo -e "\n"
./hive --sim 'ethereum/eest/consume-engine' --client erigon 2>&1 | tee output.log || {
if [ $? -gt 0 ]; then
echo "Exitcode gt 0"
fi
}
status_line=$(tail -2 output.log | head -1 | sed -r "s/\x1B\[[0-9;]*[a-zA-Z]//g")
suites=$(echo "$status_line" | sed -n 's/.*suites=\([0-9]*\).*/\1/p')
if [ -z "$suites" ]; then
status_line=$(tail -1 output.log | sed -r "s/\x1B\[[0-9;]*[a-zA-Z]//g")
suites=$(echo "$status_line" | sed -n 's/.*suites=\([0-9]*\).*/\1/p')
fi
tests=$(echo "$status_line" | sed -n 's/.*tests=\([0-9]*\).*/\1/p')
failed=$(echo "$status_line" | sed -n 's/.*failed=\([0-9]*\).*/\1/p')
echo -e "\n"
echo "----------- Results for ${1} -----------"
echo "Tests: $tests, Failed: $failed"
echo -e "\n\n============================================================"
if (( tests < 4 )); then
echo "Too few tests run for suite ${1} - ${tests} tests"
exit 1
fi
if (( failed*2 > tests )); then
echo "Too many failures for suite ${1} - ${failed} failed out of ${tests}"
exit 1
fi
}
run_suite eest/consume-engine
Loading

0 comments on commit 32510ea

Please sign in to comment.