From 26f50099f4f444c22056aebed44077324c227e9d Mon Sep 17 00:00:00 2001 From: GalaIO Date: Wed, 10 Apr 2024 14:31:23 +0800 Subject: [PATCH 1/5] blobtx: fix some found issues; (#2383) --- core/data_availability_test.go | 86 ++++++++++++++++++++++++++++++++++ core/rawdb/accessors_chain.go | 2 +- core/rawdb/chain_freezer.go | 38 ++++++++++++++- core/rawdb/freezer.go | 2 +- 4 files changed, 124 insertions(+), 4 deletions(-) diff --git a/core/data_availability_test.go b/core/data_availability_test.go index e6963a2615..2269aee232 100644 --- a/core/data_availability_test.go +++ b/core/data_availability_test.go @@ -1,9 +1,12 @@ package core import ( + "crypto/rand" "math/big" "testing" + "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" + gokzg4844 "github.com/crate-crypto/go-kzg-4844" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto/kzg4844" @@ -247,6 +250,49 @@ func TestCheckDataAvailableInBatch(t *testing.T) { } } +func BenchmarkEmptySidecarDAChecking(b *testing.B) { + hr := NewMockDAHeaderReader(params.ParliaTestChainConfig) + block := types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(1), + }).WithBody(types.Transactions{ + createMockDATx(hr.Config(), emptySidecar()), + createMockDATx(hr.Config(), emptySidecar()), + createMockDATx(hr.Config(), emptySidecar()), + createMockDATx(hr.Config(), emptySidecar()), + createMockDATx(hr.Config(), emptySidecar()), + createMockDATx(hr.Config(), emptySidecar()), + }, nil) + block = block.WithSidecars(collectBlobsFromTxs(block.Header(), block.Transactions())) + b.ResetTimer() + for i := 0; i < b.N; i++ { + IsDataAvailable(hr, block) + } +} + +func BenchmarkRandomSidecarDAChecking(b *testing.B) { + hr := NewMockDAHeaderReader(params.ParliaTestChainConfig) + const count = 10 + blocks := make([]*types.Block, count) + for i := 0; i < len(blocks); i++ { + block := types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(1), + }).WithBody(types.Transactions{ + createMockDATx(hr.Config(), randomSidecar()), + createMockDATx(hr.Config(), randomSidecar()), + createMockDATx(hr.Config(), randomSidecar()), + createMockDATx(hr.Config(), randomSidecar()), + createMockDATx(hr.Config(), randomSidecar()), + createMockDATx(hr.Config(), randomSidecar()), + }, nil) + block = block.WithSidecars(collectBlobsFromTxs(block.Header(), block.Transactions())) + blocks[i] = block + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + IsDataAvailable(hr, blocks[i%count]) + } +} + func collectBlobsFromTxs(header *types.Header, txs types.Transactions) types.BlobSidecars { sidecars := make(types.BlobSidecars, 0, len(txs)) for i, tx := range txs { @@ -348,3 +394,43 @@ func createMockDATx(config *params.ChainConfig, sidecar *types.BlobTxSidecar) *t } return types.NewTx(tx) } + +func randFieldElement() [32]byte { + bytes := make([]byte, 32) + _, err := rand.Read(bytes) + if err != nil { + panic("failed to get random field element") + } + var r fr.Element + r.SetBytes(bytes) + + return gokzg4844.SerializeScalar(r) +} + +func randBlob() kzg4844.Blob { + var blob kzg4844.Blob + for i := 0; i < len(blob); i += gokzg4844.SerializedScalarSize { + fieldElementBytes := randFieldElement() + copy(blob[i:i+gokzg4844.SerializedScalarSize], fieldElementBytes[:]) + } + return blob +} + +func randomSidecar() *types.BlobTxSidecar { + blob := randBlob() + commitment, _ := kzg4844.BlobToCommitment(blob) + proof, _ := kzg4844.ComputeBlobProof(blob, commitment) + return &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{blob}, + Commitments: []kzg4844.Commitment{commitment}, + Proofs: []kzg4844.Proof{proof}, + } +} + +func emptySidecar() *types.BlobTxSidecar { + return &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + } +} diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index baa3df7ef6..fcbf2ff4a6 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -808,7 +808,7 @@ func WriteAncientBlocksWithBlobs(db ethdb.AncientWriter, blocks []*types.Block, break } } - log.Info("WriteAncientBlocks", "startAt", blocks[0].Number(), "cancunIndex", cancunIndex, "len", len(blocks)) + log.Debug("WriteAncientBlocks", "startAt", blocks[0].Number(), "cancunIndex", cancunIndex, "len", len(blocks)) var ( tdSum = new(big.Int).Set(td) diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 5f6cd15ad6..323a7eb54c 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -17,6 +17,7 @@ package rawdb import ( + "errors" "fmt" "math/big" "sync" @@ -42,6 +43,10 @@ const ( freezerBatchLimit = 30000 ) +var ( + missFreezerEnvErr = errors.New("missing freezer env error") +) + // chainFreezer is a wrapper of freezer with additional chain freezing feature. // The background thread will keep moving ancient chain segments from key-value // database to flat files for saving space on live database. @@ -119,6 +124,20 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { return } } + + // check freezer env first, it must wait a while when the env is necessary + err := f.checkFreezerEnv() + if err == missFreezerEnvErr { + log.Warn("Freezer need related env, may wait for a while", "err", err) + backoff = true + continue + } + if err != nil { + log.Error("Freezer check FreezerEnv err", "err", err) + backoff = true + continue + } + // Retrieve the freezing threshold. hash := ReadHeadBlockHash(nfdb) if hash == (common.Hash{}) { @@ -278,7 +297,7 @@ func (f *chainFreezer) tryPruneBlobAncientTable(env *ethdb.FreezerEnv, num uint6 log.Error("Cannot prune blob ancient", "block", num, "expectTail", expectTail, "err", err) return } - log.Info("Chain freezer prune useless blobs, now ancient data is", "from", expectTail, "to", num, "cost", common.PrettyDuration(time.Since(start))) + log.Debug("Chain freezer prune useless blobs, now ancient data is", "from", expectTail, "to", num, "cost", common.PrettyDuration(time.Since(start))) } func getBlobExtraReserveFromEnv(env *ethdb.FreezerEnv) uint64 { @@ -290,7 +309,7 @@ func getBlobExtraReserveFromEnv(env *ethdb.FreezerEnv) uint64 { func (f *chainFreezer) freezeRangeWithBlobs(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) { defer func() { - log.Info("freezeRangeWithBlobs", "from", number, "to", limit, "err", err) + log.Debug("freezeRangeWithBlobs", "from", number, "to", limit, "err", err) }() lastHash := ReadCanonicalHash(nfdb, limit) if lastHash == (common.Hash{}) { @@ -413,6 +432,21 @@ func (f *chainFreezer) SetupFreezerEnv(env *ethdb.FreezerEnv) error { return nil } +func (f *chainFreezer) checkFreezerEnv() error { + _, exist := f.freezeEnv.Load().(*ethdb.FreezerEnv) + if exist { + return nil + } + blobFrozen, err := f.TableAncients(ChainFreezerBlobSidecarTable) + if err != nil { + return err + } + if blobFrozen > 0 { + return missFreezerEnvErr + } + return nil +} + func isCancun(env *ethdb.FreezerEnv, num *big.Int, time uint64) bool { if env == nil || env.ChainCfg == nil { return false diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index ca237dd3aa..ba1542294d 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -765,7 +765,7 @@ func (f *Freezer) ResetTable(kind string, startAt uint64, onlyEmpty bool) error f.frozen.Add(f.offset) f.tail.Add(f.offset) f.writeBatch = newFreezerBatch(f) - log.Info("Reset Table", "kind", kind, "tail", f.tables[kind].itemHidden.Load(), "frozen", f.tables[kind].items.Load()) + log.Debug("Reset Table", "kind", kind, "tail", f.tables[kind].itemHidden.Load(), "frozen", f.tables[kind].items.Load()) return nil } From a75e82367d97e3a17b538e147029a98939e5fd48 Mon Sep 17 00:00:00 2001 From: buddho Date: Wed, 10 Apr 2024 14:42:16 +0800 Subject: [PATCH 2/5] fix: ensure empty withdrawals after cancun (#2384) --- consensus/parlia/parlia.go | 4 +--- core/block_validator.go | 4 ++-- core/chain_makers.go | 3 +++ core/types/block.go | 4 ++-- eth/downloader/queue.go | 6 +++--- eth/fetcher/block_fetcher.go | 5 ++++- eth/protocols/eth/handlers.go | 2 +- graphql/graphql.go | 2 +- miner/worker.go | 4 ++++ 9 files changed, 21 insertions(+), 13 deletions(-) diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 9b07fd9bb4..911a04359c 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -599,9 +599,7 @@ func (p *Parlia) verifyHeader(chain consensus.ChainHeaderReader, header *types.H switch { case header.ParentBeaconRoot != nil: return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) - // types.EmptyWithdrawalsHash represents a empty value when EIP-4895 enabled, - // here, EIP-4895 still be disabled, value expected to be `types.EmptyWithdrawalsHash` is only to feet the demand of rlp encode/decode - case header.WithdrawalsHash == nil || *header.WithdrawalsHash != types.EmptyWithdrawalsHash: + case !header.EmptyWithdrawalsHash(): return errors.New("header has wrong WithdrawalsHash") } if err := eip4844.VerifyEIP4844Header(parent, header); err != nil { diff --git a/core/block_validator.go b/core/block_validator.go index 8bd5071f12..d15e2cd786 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -96,7 +96,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { }, func() error { // Withdrawals are present after the Shanghai fork. - if !header.EmptyWithdrawalsHash() { + if header.WithdrawalsHash != nil { // Withdrawals list must be present in body after Shanghai. if block.Withdrawals() == nil { return errors.New("missing withdrawals in block body") @@ -104,7 +104,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash { return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash) } - } else if len(block.Withdrawals()) != 0 { // Withdrawals turn into empty from nil when BlockBody has Sidecars + } else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars // Withdrawals are not allowed prior to shanghai fork return errors.New("withdrawals present in block body") } diff --git a/core/chain_makers.go b/core/chain_makers.go index e3cf61dc63..0592210dba 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -372,6 +372,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if err != nil { panic(err) } + if block.Header().EmptyWithdrawalsHash() { + block = block.WithWithdrawals(make([]*types.Withdrawal, 0)) + } if config.IsCancun(block.Number(), block.Time()) { for _, s := range b.sidecars { s.BlockNumber = block.Number() diff --git a/core/types/block.go b/core/types/block.go index e947792c1e..422363b32f 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -193,9 +193,9 @@ func (h *Header) EmptyReceipts() bool { return h.ReceiptHash == EmptyReceiptsHash } -// EmptyWithdrawalsHash returns true if there are no WithdrawalsHash for this header/block. +// EmptyWithdrawalsHash returns true if the WithdrawalsHash is EmptyWithdrawalsHash. func (h *Header) EmptyWithdrawalsHash() bool { - return h.WithdrawalsHash == nil || *h.WithdrawalsHash == EmptyWithdrawalsHash + return h.WithdrawalsHash != nil && *h.WithdrawalsHash == EmptyWithdrawalsHash } // Body is a simple (mutable, non-safe) data container for storing and moving diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index b97cfdec6d..72d21c329e 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -81,7 +81,7 @@ func newFetchResult(header *types.Header, fastSync bool, pid string) *fetchResul } if !header.EmptyBody() { item.pending.Store(item.pending.Load() | (1 << bodyType)) - } else if !header.EmptyWithdrawalsHash() { + } else if header.WithdrawalsHash != nil { item.Withdrawals = make(types.Withdrawals, 0) } if fastSync && !header.EmptyReceipts() { @@ -788,9 +788,9 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, txListH if uncleListHashes[index] != header.UncleHash { return errInvalidBody } - if header.EmptyWithdrawalsHash() { + if header.WithdrawalsHash == nil { // nil hash means that withdrawals should not be present in body - if len(withdrawalLists[index]) != 0 { + if withdrawalLists[index] != nil { return errInvalidBody } } else { // non-nil hash: body must have withdrawals diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index e013e3b29d..4cd6d5f37e 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -576,7 +576,7 @@ func (f *BlockFetcher) loop() { select { case res := <-resCh: res.Done <- nil - // Ignoring withdrawals here, since the block fetcher is not used post-merge. + // Ignoring withdrawals here, will set it to empty later if EmptyWithdrawalsHash in header. txs, uncles, _, sidecars := res.Res.(*eth.BlockBodiesResponse).Unpack() f.FilterBodies(peer, txs, uncles, sidecars, time.Now()) @@ -723,6 +723,9 @@ func (f *BlockFetcher) loop() { matched = true if f.getBlock(hash) == nil { block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i]) + if block.Header().EmptyWithdrawalsHash() { + block = block.WithWithdrawals(make([]*types.Withdrawal, 0)) + } block = block.WithSidecars(task.sidecars[i]) block.ReceivedAt = task.time blocks = append(blocks, block) diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 992449be69..b028f50637 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -367,7 +367,7 @@ func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { txsHashes[i] = types.DeriveSha(types.Transactions(body.Transactions), hasher) uncleHashes[i] = types.CalcUncleHash(body.Uncles) // Withdrawals may be not nil, but a empty value when Sidecars not empty - if len(body.Withdrawals) > 0 { + if body.Withdrawals != nil { withdrawalHashes[i] = types.DeriveSha(types.Withdrawals(body.Withdrawals), hasher) } } diff --git a/graphql/graphql.go b/graphql/graphql.go index cf4d7a34ca..fd782206fb 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -1053,7 +1053,7 @@ func (b *Block) Withdrawals(ctx context.Context) (*[]*Withdrawal, error) { return nil, err } // Pre-shanghai blocks - if block.Header().EmptyWithdrawalsHash() { + if block.Header().WithdrawalsHash == nil { return nil, nil } ret := make([]*Withdrawal, 0, len(block.Withdrawals())) diff --git a/miner/worker.go b/miner/worker.go index cdd47e2bc6..09d7fedf03 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1367,6 +1367,10 @@ func (w *worker) commit(env *environment, interval func(), update bool, start ti // env.receipts = receipts finalizeBlockTimer.UpdateSince(finalizeStart) + if block.Header().EmptyWithdrawalsHash() { + block = block.WithWithdrawals(make([]*types.Withdrawal, 0)) + } + // If Cancun enabled, sidecars can't be nil then. if w.chainConfig.IsCancun(env.header.Number, env.header.Time) && env.sidecars == nil { env.sidecars = make(types.BlobSidecars, 0) From f4bad204471e10fd9aafbae2be5a4b1127c9347f Mon Sep 17 00:00:00 2001 From: clonefetch <166312854+clonefetch@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:06:12 +0800 Subject: [PATCH 3/5] docs: remove repetitive words (#2381) --- accounts/external/backend.go | 2 +- cmd/geth/snapshot.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/accounts/external/backend.go b/accounts/external/backend.go index 84f33d237e..12de26b09e 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -163,7 +163,7 @@ func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, d hexutil.Encode(data)); err != nil { return nil, err } - // If V is on 27/28-form, convert to to 0/1 for Clique and Parlia + // If V is on 27/28-form, convert to 0/1 for Clique and Parlia if (mimeType == accounts.MimetypeClique || mimeType == accounts.MimetypeParlia) && (res[64] == 27 || res[64] == 28) { res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique and Parlia use } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 48a1ae0d11..d00fc9523f 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -92,7 +92,7 @@ WARNING: it's only supported in hash mode(--state.scheme=hash)". geth offline prune-block for block data in ancientdb. The amount of blocks expected for remaining after prune can be specified via block-amount-reserved in this command, will prune and only remain the specified amount of old block data in ancientdb. -the brief workflow is to backup the the number of this specified amount blocks backward in original ancientdb +the brief workflow is to backup the number of this specified amount blocks backward in original ancientdb into new ancient_backup, then delete the original ancientdb dir and rename the ancient_backup to original one for replacement, finally assemble the statedb and new ancientDb together. The purpose of doing it is because the block data will be moved into the ancient store when it From d7836bfe989da8609322603f0eb5b5d2a65b4bc8 Mon Sep 17 00:00:00 2001 From: buddho Date: Wed, 10 Apr 2024 19:46:34 +0800 Subject: [PATCH 4/5] fix: set withdrawals properly when fetching empty block (#2386) --- eth/fetcher/block_fetcher.go | 3 +++ eth/protocols/eth/handlers.go | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 4cd6d5f37e..26c58d7143 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -639,6 +639,9 @@ func (f *BlockFetcher) loop() { log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) block := types.NewBlockWithHeader(header) + if block.Header().EmptyWithdrawalsHash() { + block = block.WithWithdrawals(make([]*types.Withdrawal, 0)) + } block.ReceivedAt = task.time complete = append(complete, block) diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index b028f50637..583dd2f67c 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -366,7 +366,6 @@ func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { for i, body := range res.BlockBodiesResponse { txsHashes[i] = types.DeriveSha(types.Transactions(body.Transactions), hasher) uncleHashes[i] = types.CalcUncleHash(body.Uncles) - // Withdrawals may be not nil, but a empty value when Sidecars not empty if body.Withdrawals != nil { withdrawalHashes[i] = types.DeriveSha(types.Withdrawals(body.Withdrawals), hasher) } From 009df5a1213c1be4f9eb455c4b863a95a4f59d2d Mon Sep 17 00:00:00 2001 From: Eric <45141191+zlacfzy@users.noreply.github.com> Date: Wed, 10 Apr 2024 20:13:05 +0800 Subject: [PATCH 5/5] internal/api: return empty array rather than nil (#2385) --- internal/ethapi/api.go | 4 ++-- .../testdata/eth_getBlobSidecars-block-with-blob-tx.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index fc46f6591b..c03fdd9323 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1018,7 +1018,7 @@ func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.B return nil, nil } blobSidecars, err := s.b.GetBlobSidecars(ctx, header.Hash()) - if err != nil || len(blobSidecars) == 0 { + if err != nil || blobSidecars == nil { return nil, nil } result := make([]map[string]interface{}, len(blobSidecars)) @@ -1040,7 +1040,7 @@ func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common. return nil, nil } blobSidecars, err := s.b.GetBlobSidecars(ctx, blockHash) - if err != nil || len(blobSidecars) == 0 { + if err != nil || blobSidecars == nil || len(blobSidecars) == 0 { return nil, nil } for _, sidecar := range blobSidecars { diff --git a/internal/ethapi/testdata/eth_getBlobSidecars-block-with-blob-tx.json b/internal/ethapi/testdata/eth_getBlobSidecars-block-with-blob-tx.json index ec747fa47d..0637a088a0 100644 --- a/internal/ethapi/testdata/eth_getBlobSidecars-block-with-blob-tx.json +++ b/internal/ethapi/testdata/eth_getBlobSidecars-block-with-blob-tx.json @@ -1 +1 @@ -null \ No newline at end of file +[] \ No newline at end of file