Skip to content

Commit

Permalink
Merge tag 'v1.2.2' into release/polygon-1.x-fh2.3
Browse files Browse the repository at this point in the history 
# Conflicts:
#	packaging/templates/package_scripts/control
#	packaging/templates/package_scripts/control.arm64
#	packaging/templates/package_scripts/control.profile.amd64
#	packaging/templates/package_scripts/control.profile.arm64
#	packaging/templates/package_scripts/control.validator
#	packaging/templates/package_scripts/control.validator.arm64
#	params/version.go
  • Loading branch information
@maoueh
maoueh committed Jan 16, 2024
2 parents 610fd42 + 70bebc9 commit 4c7985c
Show file tree
Hide file tree
Showing 12 changed files with 20 additions and 71 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ jobs:

- name: Test
run: make test

- uses: actions/upload-artifact@v2
with:
name: unitTest-coverage
Expand Down Expand Up @@ -217,7 +217,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: '16.17.1'
node-version: '18.19.0'
cache: 'npm'
cache-dependency-path: |
matic-cli/package-lock.json
Expand All @@ -231,7 +231,7 @@ jobs:
npm install --prefer-offline --no-audit --progress=false
mkdir devnet
cd devnet
../bin/matic-cli setup devnet -c ../../bor/.github/matic-cli-config.yml
../bin/matic-cli.js setup devnet -c ../../bor/.github/matic-cli-config.yml
- name: Launch devnet
run: |
Expand Down
52 changes: 0 additions & 52 deletions core/state/pruner/pruner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,12 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
if headBlock == nil {
return nil, errors.New("failed to load head block")
}

snapconfig := snapshot.Config{
CacheSize: 256,
Recovery: false,
NoBuild: true,
AsyncBuild: false,
}

snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
if err != nil {
return nil, err // The relevant snapshot(s) might not exist
Expand All @@ -103,12 +101,10 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
log.Warn("Sanitizing bloomfilter size", "provided(MB)", config.BloomSize, "updated(MB)", 256)
config.BloomSize = 256
}

stateBloom, err := newStateBloomWithSize(config.BloomSize)
if err != nil {
return nil, err
}

return &Pruner{
config: config,
chainHeader: headBlock.Header(),
Expand All @@ -134,7 +130,6 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
batch = maindb.NewBatch()
iter = maindb.NewIterator(nil, nil)
)

for iter.Next() {
key := iter.Key()

Expand All @@ -148,34 +143,28 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
if isCode {
checkKey = codeKey
}

if _, exist := middleStateRoots[common.BytesToHash(checkKey)]; exist {
log.Debug("Forcibly delete the middle state roots", "hash", common.BytesToHash(checkKey))
} else {
if stateBloom.Contain(checkKey) {
continue
}
}

count += 1
size += common.StorageSize(len(key) + len(iter.Value()))
batch.Delete(key)

var eta time.Duration // Realistically will never remain uninited

if done := binary.BigEndian.Uint64(key[:8]); done > 0 {
var (
left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero
)

eta = time.Duration(left/speed) * time.Millisecond
}

if time.Since(logged) > 8*time.Second {
log.Info("Pruning state data", "nodes", count, "size", size,
"elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta))

logged = time.Now()
}
// Recreate the iterator after every batch commit in order
Expand All @@ -189,12 +178,10 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
}
}
}

if batch.ValueSize() > 0 {
batch.Write()
batch.Reset()
}

iter.Release()
log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart)))

Expand All @@ -221,36 +208,29 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
// Note for small pruning, the compaction is skipped.
if count >= rangeCompactionThreshold {
cstart := time.Now()

for b := 0x00; b <= 0xf0; b += 0x10 {
var (
start = []byte{byte(b)}
end = []byte{byte(b + 0x10)}
)

if b == 0xf0 {
end = nil
}

log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart)))

if err := maindb.Compact(start, end); err != nil {
log.Error("Database compaction failed", "error", err)
return err
}
}
log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart)))
}

log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start)))

return nil
}

// Prune deletes all historical state nodes except the nodes belong to the
// specified state version. If user doesn't specify the state version, use
// the bottom-most snapshot diff layer as the target.
// nolint:nestif
func (p *Pruner) Prune(root common.Hash) error {
// If the state bloom filter is already committed previously,
// reuse it for pruning instead of generating a new one. It's
Expand All @@ -260,7 +240,6 @@ func (p *Pruner) Prune(root common.Hash) error {
if err != nil {
return err
}

if stateBloomRoot != (common.Hash{}) {
return RecoverPruning(p.config.Datadir, p.db, p.config.Cachedir)
}
Expand Down Expand Up @@ -299,23 +278,18 @@ func (p *Pruner) Prune(root common.Hash) error {
// state available, but we don't want to use the topmost state
// as the pruning target.
var found bool

for i := len(layers) - 2; i >= 2; i-- {
if rawdb.HasLegacyTrieNode(p.db, layers[i].Root()) {
root = layers[i].Root()
found = true

log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i)

break
}
}

if !found {
if len(layers) > 0 {
return errors.New("no snapshot paired state")
}

return fmt.Errorf("associated state[%x] is not present", root)
}
} else {
Expand All @@ -334,18 +308,15 @@ func (p *Pruner) Prune(root common.Hash) error {
// All the state roots of the middle layer should be forcibly pruned,
// otherwise the dangling state will be left.
middleRoots := make(map[common.Hash]struct{})

for _, layer := range layers {
if layer.Root() == root {
break
}

middleRoots[layer.Root()] = struct{}{}
}
// Traverse the target state, re-construct the whole state trie and
// commit to the given bloom filter.
start := time.Now()

if err := snapshot.GenerateTrie(p.snaptree, root, p.db, p.stateBloom); err != nil {
return err
}
Expand All @@ -354,17 +325,13 @@ func (p *Pruner) Prune(root common.Hash) error {
if err := extractGenesis(p.db, p.stateBloom); err != nil {
return err
}

filterName := bloomFilterName(p.config.Datadir, root)

log.Info("Writing state bloom to disk", "name", filterName)

if err := p.stateBloom.Commit(filterName, filterName+stateBloomFileTempSuffix); err != nil {
return err
}

log.Info("State bloom filter committed", "name", filterName)

return prune(p.snaptree, root, p.db, p.stateBloom, filterName, middleRoots, start)
}

Expand All @@ -380,11 +347,9 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err
if err != nil {
return err
}

if stateBloomPath == "" {
return nil // nothing to recover
}

headBlock := rawdb.ReadHeadBlock(db)
if headBlock == nil {
return errors.New("failed to load head block")
Expand All @@ -403,17 +368,14 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err
NoBuild: true,
AsyncBuild: false,
}

snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
if err != nil {
return err // The relevant snapshot(s) might not exist
}

stateBloom, err := NewStateBloomFromDisk(stateBloomPath)
if err != nil {
return err
}

log.Info("Loaded state bloom filter", "path", stateBloomPath)

// Before start the pruning, delete the clean trie cache first.
Expand All @@ -429,21 +391,17 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err
layers = snaptree.Snapshots(headBlock.Root(), 128, true)
middleRoots = make(map[common.Hash]struct{})
)

for _, layer := range layers {
if layer.Root() == stateBloomRoot {
found = true
break
}

middleRoots[layer.Root()] = struct{}{}
}

if !found {
log.Error("Pruning target state is not existent")
return errors.New("non-existent target state")
}

return prune(snaptree, stateBloomRoot, db, stateBloom, stateBloomPath, middleRoots, time.Now())
}

Expand All @@ -454,12 +412,10 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesisHash == (common.Hash{}) {
return errors.New("missing genesis hash")
}

genesis := rawdb.ReadBlock(db, genesisHash, 0)
if genesis == nil {
return errors.New("missing genesis block")
}

t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
if err != nil {
return err
Expand All @@ -483,10 +439,8 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil {
return err
}

if acc.Root != types.EmptyRootHash {
id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)

storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
if err != nil {
return err
Expand All @@ -502,18 +456,15 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
stateBloom.Put(hash.Bytes(), nil)
}
}

if storageIter.Error() != nil {
return storageIter.Error()
}
}

if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash.Bytes()) {
stateBloom.Put(acc.CodeHash, nil)
}
}
}

return accIter.Error()
}

Expand All @@ -526,7 +477,6 @@ func isBloomFilter(filename string) (bool, common.Hash) {
if strings.HasPrefix(filename, stateBloomFilePrefix) && strings.HasSuffix(filename, stateBloomFileSuffix) {
return true, common.HexToHash(filename[len(stateBloomFilePrefix)+1 : len(filename)-len(stateBloomFileSuffix)-1])
}

return false, common.Hash{}
}

Expand All @@ -535,7 +485,6 @@ func findBloomFilter(datadir string) (string, common.Hash, error) {
stateBloomPath string
stateBloomRoot common.Hash
)

if err := filepath.Walk(datadir, func(path string, info os.FileInfo, err error) error {
if info != nil && !info.IsDir() {
ok, root := isBloomFilter(path)
Expand All @@ -548,7 +497,6 @@ func findBloomFilter(datadir string) (string, common.Hash, error) {
}); err != nil {
return "", common.Hash{}, err
}

return stateBloomPath, stateBloomRoot, nil
}

Expand Down
1 change: 1 addition & 0 deletions eth/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
TrieTimeLimit: config.TrieTimeout,
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
TriesInMemory: config.TriesInMemory,
}
)

Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,11 @@ require (
github.com/urfave/cli/v2 v2.24.1
go.uber.org/atomic v1.11.0
go.uber.org/automaxprocs v1.5.2
golang.org/x/crypto v0.14.0
golang.org/x/crypto v0.17.0
golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
golang.org/x/sync v0.3.0
golang.org/x/sys v0.13.0
golang.org/x/text v0.13.0
golang.org/x/sys v0.15.0
golang.org/x/text v0.14.0
golang.org/x/time v0.3.0
golang.org/x/tools v0.10.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
Expand Down
12 changes: 6 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2260,8 +2260,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -2622,8 +2622,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down Expand Up @@ -2655,8 +2655,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
2 changes: 1 addition & 1 deletion packaging/templates/package_scripts/control
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Source: bor
Version: 1.2.1
Version: 1.2.2
Section: develop
Priority: standard
Maintainer: Polygon <[email protected]>
Expand Down
Loading

0 comments on commit 4c7985c

Please sign in to comment.