merge with main

.github/workflows/ci-cd-main-branch-docker-images.yml

@@ -20,19 +20,9 @@ on:
   workflow_dispatch:
 
 jobs:
-  define_matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      os: ${{ steps.os.outputs.os }}
-
-    steps:
-     - name: Define os
-       id: os
-       run: echo 'os=ubuntu-latest' >> "$GITHUB_OUTPUT"
 
   Build:
-    needs: define_matrix
-    runs-on: ${{ needs.define_matrix.outputs.os }}
+    runs-on: ubuntu-latest
     timeout-minutes: 45
     outputs:
       docker_build_tag: ${{ steps.built_tag_export.outputs.docker_build_tag }}
@@ -167,11 +157,3 @@ jobs:
               fi
               echo "Done."
             done
-
-  run-kurtosis-assertoor:
-    needs: [define_matrix, Build]
-    uses: erigontech/erigon/.github/workflows/test-kurtosis-assertoor.yml@main
-    with:
-      checkout_ref: ${{ github.sha }}
-      os: ${{ needs.define_matrix.outputs.os }}
-      docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }}

.github/workflows/test-integration-erigon.yml

@@ -1,13 +1,12 @@
-name: Integration tests - ERIGON 
+name: Integration tests - ERIGON
 on:
   push:
     branches:
-      - test-integration-michele
       - main
       - 'release/**'
   pull_request:
     branches:
-      - main
+      - main 
     types:
       - opened
       - reopened

Dockerfile.release

@@ -1,6 +1,8 @@
 ARG RELEASE_DOCKER_BASE_IMAGE="alpine:3.20.1" \
     CI_CD_MAIN_BUILDER_IMAGE="golang:1.22-bookworm" \
     CI_CD_MAIN_TARGET_BASE_IMAGE="alpine:3.20.1" \
+    UID_ERIGON=1000 \
+    GID_ERIGON=1000 \
     EXPOSED_PORTS="8545 \
        8551 \
        8546 \
@@ -30,21 +32,25 @@ FROM ${RELEASE_DOCKER_BASE_IMAGE} AS release
 
 ARG USER=erigon \
     GROUP=erigon \
+    UID_ERIGON \
+    GID_ERIGON \
     APPLICATION \
     EXPOSED_PORTS
 
+STOPSIGNAL 2
+
 RUN --mount=type=bind,from=temporary,source=/tmp/${APPLICATION},target=/tmp/${APPLICATION} \
     apk add --no-cache ca-certificates tzdata && \
-    addgroup ${GROUP} && \
-    adduser -D -h /home/${USER} -G ${GROUP} ${USER} && \
+    addgroup -g ${GID_ERIGON} ${GROUP} && \
+    adduser -D -u ${UID_ERIGON} -h /home/${USER} -G ${GROUP} ${USER} && \
     install -d -o ${USER} -g ${GROUP} /home/${USER}/.local /home/${USER}/.local/share /home/${USER}/.local/share/erigon && \
-    install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/erigon /usr/local/bin/ && \
-    install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/integration /usr/local/bin/ && \
-    install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/diag /usr/local/bin/ && \
-    install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/sentry /usr/local/bin/ && \
-    install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/txpool /usr/local/bin/ && \
-    install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/downloader /usr/local/bin/ && \
-    install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/rpcdaemon /usr/local/bin/
+    install -o root -g root /tmp/${APPLICATION}/erigon /usr/local/bin/ && \
+    install -o root -g root /tmp/${APPLICATION}/integration /usr/local/bin/ && \
+    install -o root -g root /tmp/${APPLICATION}/diag /usr/local/bin/ && \
+    install -o root -g root /tmp/${APPLICATION}/sentry /usr/local/bin/ && \
+    install -o root -g root /tmp/${APPLICATION}/txpool /usr/local/bin/ && \
+    install -o root -g root /tmp/${APPLICATION}/downloader /usr/local/bin/ && \
+    install -o root -g root /tmp/${APPLICATION}/rpcdaemon /usr/local/bin/
 
 VOLUME [ "/home/${USER}" ]
 WORKDIR /home/${USER}
@@ -90,4 +96,4 @@ EXPOSE ${EXPOSED_PORTS}
 
 ENTRYPOINT [ "/usr/local/bin/erigon" ]
 
-### End of CI-CD : main branch docker image publishing for each new commit id
+### End of CI-CD : main branch docker image publishing for each new commit id

README.md

@@ -207,7 +207,7 @@ du -hsc /erigon/snapshots/*
 - **Store most of data in immutable files (segments/snapshots):**
     - can symlink/mount latest state to fast drive and history to cheap drive
     - `chaindata` is less than `15gb`. It's ok to `rm -rf chaindata`. (to prevent grow: recommend `--batchSize <= 1G`)
-- **`--prune` flags changed**: see `--prune.mode` (default: `archive`, full: `full`, EIP-4444: `minimal`)
+- **`--prune` flags changed**: see `--prune.mode` (default: `full`, archive: `archive`, EIP-4444: `minimal`)
 - **Other changes:**
     - ExecutionStage included many E2 stages: stage_hash_state, stage_trie, log_index, history_index, trace_index
     - Restart doesn't loose much partial progress: `--sync.loop.block.limit=5_000` enabled by default

accounts/abi/bind/template.go

@@ -530,6 +530,10 @@ var (
 			Raw types.Log // Blockchain specific contextual infos
 		}
 
+		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) {{.Normalized.Name}}EventID() libcommon.Hash {
+   			return libcommon.HexToHash("0x{{printf "%x" .Original.ID}}")
+  		}
+
 		// Filter{{.Normalized.Name}} is a free log retrieval operation binding the contract event 0x{{printf "%x" .Original.ID}}.
 		//
 		// Solidity: {{.Original.String}}

cl/beacon/handler/block_production.go

@@ -148,6 +148,8 @@ func (a *ApiHandler) GetEthV1ValidatorAttestationData(
 	if err != nil {
 		return nil, beaconhttp.NewEndpointError(http.StatusBadRequest, err)
 	}
+	start := time.Now()
+
 	// wait until the head state is at the target slot or later
 	err = a.waitUntilHeadStateAtEpochIsReadyOrCountAsMissed(r.Context(), a.syncedData, *slot/a.beaconChainCfg.SlotsPerEpoch)
 	if err != nil {
@@ -179,6 +181,13 @@ func (a *ApiHandler) GetEthV1ValidatorAttestationData(
 	if err != nil {
 		log.Warn("Failed to get attestation data", "err", err)
 	}
+
+	defer func() {
+		a.logger.Debug("Produced Attestation", "slot", *slot,
+			"committee_index", *committeeIndex, "cached", ok, "beacon_block_root",
+			attestationData.BeaconBlockRoot, "duration", time.Since(start))
+	}()
+
 	if ok {
 		return newBeaconResponse(attestationData), nil
 	}

cl/clparams/config.go

@@ -66,6 +66,7 @@ type CaplinConfig struct {
 	SentinelPort           uint64
 	SubscribeAllTopics     bool
 	MaxPeerCount           uint64
+	EnableUPnP             bool
 	// Erigon Sync
 	LoopBlockLimit uint64
 	// Beacon API router configuration

cl/phase1/forkchoice/fork_graph/fork_graph_disk.go

@@ -113,8 +113,8 @@ type forkGraphDisk struct {
 	beaconCfg   *clparams.BeaconChainConfig
 	genesisTime uint64
 	// highest block seen
-	highestSeen, anchorSlot uint64
-	lowestAvailableBlock    atomic.Uint64
+	anchorSlot           uint64
+	lowestAvailableBlock atomic.Uint64
 
 	newestLightClientUpdate atomic.Value
 	// the lightclientUpdates leaks memory, but it's not a big deal since new data is added every 27 hours.
@@ -169,8 +169,13 @@ func (f *forkGraphDisk) AnchorSlot() uint64 {
 	return f.anchorSlot
 }
 
+func (f *forkGraphDisk) isBlockRootTheCurrentState(blockRoot libcommon.Hash) bool {
+	blockRootState, _ := f.currentState.BlockRoot()
+	return blockRoot == blockRootState
+}
+
 // Add a new node and edge to the graph
-func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, fullValidation bool) (*state.CachingBeaconState, ChainSegmentInsertionResult, error) {
+func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, fullValidation, shallowImport bool) (*state.CachingBeaconState, ChainSegmentInsertionResult, error) {
 	block := signedBlock.Block
 	blockRoot, err := block.HashSSZ()
 	if err != nil {
@@ -187,10 +192,17 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock,
 		return nil, BelowAnchor, nil
 	}
 
-	newState, err := f.getState(block.ParentRoot, false, true)
-	if err != nil {
-		return nil, LogisticError, fmt.Errorf("AddChainSegment: %w, parentRoot; %x", err, block.ParentRoot)
+	isBlockRootTheCurrentState := f.isBlockRootTheCurrentState(blockRoot)
+	var newState *state.CachingBeaconState
+	if isBlockRootTheCurrentState {
+		newState = f.currentState
+	} else {
+		newState, err = f.getState(block.ParentRoot, false, true)
+		if err != nil {
+			return nil, LogisticError, fmt.Errorf("AddChainSegment: %w, parentRoot; %x", err, block.ParentRoot)
+		}
 	}
+
 	if newState == nil {
 		log.Trace("AddChainSegment: missing segment", "block", libcommon.Hash(blockRoot))
 		return nil, MissingSegment, nil
@@ -199,7 +211,7 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock,
 	parentBlock, hasParentBlock := f.getBlock(block.ParentRoot)
 
 	// Before processing the state: update the newest lightclient update.
-	if block.Version() >= clparams.AltairVersion && hasParentBlock && fullValidation && hasFinalized && f.rcfg.Beacon {
+	if block.Version() >= clparams.AltairVersion && hasParentBlock && fullValidation && hasFinalized && f.rcfg.Beacon && !isBlockRootTheCurrentState {
 		nextSyncCommitteeBranch, err := newState.NextSyncCommitteeBranch()
 		if err != nil {
 			return nil, LogisticError, err
@@ -244,14 +256,22 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock,
 
 	blockRewardsCollector := &eth2.BlockRewardsCollector{}
 
-	// Execute the state
-	if invalidBlockErr := transition.TransitionState(newState, signedBlock, blockRewardsCollector, fullValidation); invalidBlockErr != nil {
-		// Add block to list of invalid blocks
-		log.Warn("Invalid beacon block", "slot", block.Slot, "blockRoot", blockRoot, "reason", invalidBlockErr)
-		f.badBlocks.Store(libcommon.Hash(blockRoot), struct{}{})
-		//f.currentState = nil
+	if !isBlockRootTheCurrentState {
+		// Execute the state
+		if invalidBlockErr := transition.TransitionState(newState, signedBlock, blockRewardsCollector, fullValidation); invalidBlockErr != nil {
+			// Add block to list of invalid blocks
+			log.Warn("Invalid beacon block", "slot", block.Slot, "blockRoot", blockRoot, "reason", invalidBlockErr)
+			f.badBlocks.Store(libcommon.Hash(blockRoot), struct{}{})
+			//f.currentState = nil
 
-		return nil, InvalidBlock, invalidBlockErr
+			return nil, InvalidBlock, invalidBlockErr
+		}
+		f.blockRewards.Store(libcommon.Hash(blockRoot), blockRewardsCollector)
+	}
+
+	f.currentState = newState
+	if shallowImport {
+		return newState, Success, nil
 	}
 
 	// update diff storages.
@@ -261,7 +281,6 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock,
 			f.currentIndicies.add(epoch, newState.RawCurrentEpochParticipation())
 			f.previousIndicies.add(epoch, newState.RawPreviousEpochParticipation())
 		}
-		f.blockRewards.Store(libcommon.Hash(blockRoot), blockRewardsCollector)
 
 		period := f.beaconCfg.SyncCommitteePeriod(newState.Slot())
 		f.syncCommittees.Store(period, syncCommittees{
@@ -277,6 +296,7 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock,
 			f.lightclientBootstraps.Store(libcommon.Hash(blockRoot), lightclientBootstrap)
 		}
 	}
+
 	f.blocks.Store(libcommon.Hash(blockRoot), signedBlock)
 	bodyRoot, err := signedBlock.Block.Body.HashSSZ()
 	if err != nil {
@@ -294,10 +314,7 @@ func (f *forkGraphDisk) AddChainSegment(signedBlock *cltypes.SignedBeaconBlock,
 	// Lastly add checkpoints to caches as well.
 	f.currentJustifiedCheckpoints.Store(libcommon.Hash(blockRoot), newState.CurrentJustifiedCheckpoint())
 	f.finalizedCheckpoints.Store(libcommon.Hash(blockRoot), newState.FinalizedCheckpoint())
-	if newState.Slot() > f.highestSeen {
-		f.highestSeen = newState.Slot()
-	}
-	f.currentState = newState
+
 	return newState, Success, nil
 }
 

cl/phase1/forkchoice/fork_graph/fork_graph_test.go

@@ -51,20 +51,20 @@ func TestForkGraphInDisk(t *testing.T) {
 	require.NoError(t, utils.DecodeSSZSnappy(anchorState, anchor, int(clparams.Phase0Version)))
 	emitter := beaconevents.NewEventEmitter()
 	graph := NewForkGraphDisk(anchorState, afero.NewMemMapFs(), beacon_router_configuration.RouterConfiguration{}, emitter)
-	_, status, err := graph.AddChainSegment(blockA, true)
+	_, status, err := graph.AddChainSegment(blockA, true, false)
 	require.NoError(t, err)
 	require.Equal(t, status, Success)
 	// Now make blockC a bad block
 	blockC.Block.ProposerIndex = 81214459 // some invalid thing
-	_, status, err = graph.AddChainSegment(blockC, true)
+	_, status, err = graph.AddChainSegment(blockC, true, false)
 	require.Error(t, err)
 	require.Equal(t, status, InvalidBlock)
 	// Save current state hash
-	_, status, err = graph.AddChainSegment(blockB, true)
+	_, status, err = graph.AddChainSegment(blockB, true, false)
 	require.NoError(t, err)
 	require.Equal(t, status, Success)
 	// Try again with same should yield success
-	_, status, err = graph.AddChainSegment(blockB, true)
+	_, status, err = graph.AddChainSegment(blockB, true, false)
 	require.NoError(t, err)
 	require.Equal(t, status, PreValidated)
 }

cl/phase1/forkchoice/fork_graph/interface.go

@@ -35,7 +35,7 @@ import (
 * to analyze and manipulate the state of the blockchain.
  */
 type ForkGraph interface {
-	AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, fullValidation bool) (*state.CachingBeaconState, ChainSegmentInsertionResult, error)
+	AddChainSegment(signedBlock *cltypes.SignedBeaconBlock, fullValidation, shallowImport bool) (*state.CachingBeaconState, ChainSegmentInsertionResult, error)
 	GetHeader(blockRoot libcommon.Hash) (*cltypes.BeaconBlockHeader, bool)
 	GetState(blockRoot libcommon.Hash, alwaysCopy bool) (*state.CachingBeaconState, error)
 	GetCurrentJustifiedCheckpoint(blockRoot libcommon.Hash) (solid.Checkpoint, bool)

cl/phase1/forkchoice/forkchoice.go

@@ -87,14 +87,15 @@ type ForkChoiceStore struct {
 	unrealizedJustifiedCheckpoint atomic.Value
 	unrealizedFinalizedCheckpoint atomic.Value
 
-	proposerBoostRoot     atomic.Value
-	headHash              libcommon.Hash
-	headSlot              uint64
-	genesisTime           uint64
-	genesisValidatorsRoot libcommon.Hash
-	weights               map[libcommon.Hash]uint64
-	headSet               map[libcommon.Hash]struct{}
-	hotSidecars           map[libcommon.Hash][]*cltypes.BlobSidecar // Set of sidecars that are not yet processed.
+	proposerBoostRoot        atomic.Value
+	headHash                 libcommon.Hash
+	headSlot                 uint64
+	genesisTime              uint64
+	genesisValidatorsRoot    libcommon.Hash
+	weights                  map[libcommon.Hash]uint64
+	headSet                  map[libcommon.Hash]struct{}
+	hotSidecars              map[libcommon.Hash][]*cltypes.BlobSidecar // Set of sidecars that are not yet processed.
+	verifiedExecutionPayload *lru.Cache[libcommon.Hash, struct{}]
 	// childrens
 	childrens sync.Map
 
@@ -173,6 +174,11 @@ func NewForkChoiceStore(
 		Epoch: state2.Epoch(anchorState.BeaconState),
 	}
 
+	verifiedExecutionPayload, err := lru.New[libcommon.Hash, struct{}](1024)
+	if err != nil {
+		return nil, err
+	}
+
 	eth2Roots, err := lru.New[libcommon.Hash, libcommon.Hash](checkpointsPerCache)
 	if err != nil {
 		return nil, err
@@ -227,33 +233,34 @@ func NewForkChoiceStore(
 	headSet := make(map[libcommon.Hash]struct{})
 	headSet[anchorRoot] = struct{}{}
 	f := &ForkChoiceStore{
-		forkGraph:               forkGraph,
-		equivocatingIndicies:    make([]byte, anchorState.ValidatorLength(), anchorState.ValidatorLength()*2),
-		latestMessages:          newLatestMessagesStore(anchorState.ValidatorLength()),
-		eth2Roots:               eth2Roots,
-		engine:                  engine,
-		operationsPool:          operationsPool,
-		beaconCfg:               anchorState.BeaconConfig(),
-		preverifiedSizes:        preverifiedSizes,
-		finalityCheckpoints:     finalityCheckpoints,
-		totalActiveBalances:     totalActiveBalances,
-		randaoMixesLists:        randaoMixesLists,
-		randaoDeltas:            randaoDeltas,
-		headSet:                 headSet,
-		weights:                 make(map[libcommon.Hash]uint64),
-		participation:           participation,
-		emitters:                emitters,
-		genesisTime:             anchorState.GenesisTime(),
-		syncedDataManager:       syncedDataManager,
-		nextBlockProposers:      nextBlockProposers,
-		genesisValidatorsRoot:   anchorState.GenesisValidatorsRoot(),
-		hotSidecars:             make(map[libcommon.Hash][]*cltypes.BlobSidecar),
-		blobStorage:             blobStorage,
-		ethClock:                ethClock,
-		optimisticStore:         optimistic.NewOptimisticStore(),
-		validatorMonitor:        validatorMonitor,
-		probabilisticHeadGetter: probabilisticHeadGetter,
-		publicKeysRegistry:      publicKeysRegistry,
+		forkGraph:                forkGraph,
+		equivocatingIndicies:     make([]byte, anchorState.ValidatorLength(), anchorState.ValidatorLength()*2),
+		latestMessages:           newLatestMessagesStore(anchorState.ValidatorLength()),
+		eth2Roots:                eth2Roots,
+		engine:                   engine,
+		operationsPool:           operationsPool,
+		beaconCfg:                anchorState.BeaconConfig(),
+		preverifiedSizes:         preverifiedSizes,
+		finalityCheckpoints:      finalityCheckpoints,
+		totalActiveBalances:      totalActiveBalances,
+		randaoMixesLists:         randaoMixesLists,
+		randaoDeltas:             randaoDeltas,
+		headSet:                  headSet,
+		weights:                  make(map[libcommon.Hash]uint64),
+		participation:            participation,
+		emitters:                 emitters,
+		genesisTime:              anchorState.GenesisTime(),
+		syncedDataManager:        syncedDataManager,
+		nextBlockProposers:       nextBlockProposers,
+		genesisValidatorsRoot:    anchorState.GenesisValidatorsRoot(),
+		hotSidecars:              make(map[libcommon.Hash][]*cltypes.BlobSidecar),
+		blobStorage:              blobStorage,
+		ethClock:                 ethClock,
+		optimisticStore:          optimistic.NewOptimisticStore(),
+		validatorMonitor:         validatorMonitor,
+		probabilisticHeadGetter:  probabilisticHeadGetter,
+		publicKeysRegistry:       publicKeysRegistry,
+		verifiedExecutionPayload: verifiedExecutionPayload,
 	}
 	f.justifiedCheckpoint.Store(anchorCheckpoint)
 	f.finalizedCheckpoint.Store(anchorCheckpoint)