From a6523431624c004fddf0e9d004cd52650facd3b7 Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Thu, 14 Apr 2022 12:28:02 +0300 Subject: [PATCH 01/19] - call SetIdle when trying to do a snapshot when running in import-db - refactored AccountsDB constructor to use arguments --- .../disabled/processStatusHandler.go | 0 .../baseResolversContainerFactory.go | 2 +- epochStart/metachain/systemSCs_test.go | 20 +- factory/blockProcessorCreator_test.go | 17 +- factory/stateComponents.go | 53 +- genesis/process/memoryComponents.go | 24 +- integrationTests/consensus/testInitializer.go | 24 +- .../state/stateTrie/stateTrie_test.go | 24 +- integrationTests/testInitializer.go | 11 +- integrationTests/vm/testInitializer.go | 20 +- state/accountsDB.go | 125 +++-- state/accountsDB_test.go | 529 ++++++++++-------- state/errors.go | 3 + state/export_test.go | 13 + state/journalEntries_test.go | 1 - state/peerAccount.go | 2 +- state/peerAccountsDB.go | 50 +- state/peerAccountsDB_test.go | 349 +++++------- .../storagePruningManager_test.go | 24 +- state/trackableDataTrie.go | 4 +- state/trackableDataTrie_test.go | 6 +- state/userAccount.go | 2 +- testscommon/trie/statisticsMock.go | 10 +- update/genesis/import.go | 37 +- 24 files changed, 741 insertions(+), 609 deletions(-) rename {dataRetriever/factory/storageResolversContainer => common}/disabled/processStatusHandler.go (100%) diff --git a/dataRetriever/factory/storageResolversContainer/disabled/processStatusHandler.go b/common/disabled/processStatusHandler.go similarity index 100% rename from dataRetriever/factory/storageResolversContainer/disabled/processStatusHandler.go rename to common/disabled/processStatusHandler.go diff --git a/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go b/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go index 9f524d46f0f..f14d4cfdd79 100644 --- a/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go +++ b/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go @@ -10,9 +10,9 @@ import ( "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" + "github.com/ElrondNetwork/elrond-go/common/disabled" "github.com/ElrondNetwork/elrond-go/config" "github.com/ElrondNetwork/elrond-go/dataRetriever" - "github.com/ElrondNetwork/elrond-go/dataRetriever/factory/storageResolversContainer/disabled" "github.com/ElrondNetwork/elrond-go/dataRetriever/storageResolvers" "github.com/ElrondNetwork/elrond-go/process/factory" "github.com/ElrondNetwork/elrond-go/sharding" diff --git a/epochStart/metachain/systemSCs_test.go b/epochStart/metachain/systemSCs_test.go index 79254ef0415..0d38303f204 100644 --- a/epochStart/metachain/systemSCs_test.go +++ b/epochStart/metachain/systemSCs_test.go @@ -864,14 +864,24 @@ func addKeysToWaitingList( func createAccountsDB( hasher hashing.Hasher, - marshalizer marshal.Marshalizer, + marshaller marshal.Marshalizer, accountFactory state.AccountFactory, trieStorageManager common.StorageManager, ) *state.AccountsDB { - tr, _ := trie.NewTrie(trieStorageManager, marshalizer, hasher, 5) - ewl, _ := evictionWaitingList.NewEvictionWaitingList(10, testscommon.NewMemDbMock(), marshalizer) + tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, 5) + ewl, _ := evictionWaitingList.NewEvictionWaitingList(10, testscommon.NewMemDbMock(), marshaller) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) - adb, _ := state.NewAccountsDB(tr, hasher, marshalizer, accountFactory, spm, common.Normal) + + args := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(args) return adb } @@ -1313,7 +1323,7 @@ func TestSystemSCProcessor_ESDTInitShouldWork(t *testing.T) { require.Nil(t, err) require.Equal(t, 4, len(updatedContractConfig)) require.Equal(t, args.ESDTOwnerAddressBytes, updatedContractConfig[0]) - //the other config values should be unchanged + // the other config values should be unchanged for i := 1; i < len(initialContractConfig); i++ { assert.Equal(t, initialContractConfig[i], updatedContractConfig[i]) } diff --git a/factory/blockProcessorCreator_test.go b/factory/blockProcessorCreator_test.go index f3b52340017..1e19fdb6404 100644 --- a/factory/blockProcessorCreator_test.go +++ b/factory/blockProcessorCreator_test.go @@ -16,8 +16,8 @@ import ( "github.com/ElrondNetwork/elrond-go/state/storagePruningManager/disabled" "github.com/ElrondNetwork/elrond-go/storage/txcache" "github.com/ElrondNetwork/elrond-go/testscommon" - stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" "github.com/ElrondNetwork/elrond-go/testscommon/hashingMocks" + stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" "github.com/ElrondNetwork/elrond-go/trie" trieFactory "github.com/ElrondNetwork/elrond-go/trie/factory" vmcommon "github.com/ElrondNetwork/elrond-vm-common" @@ -161,17 +161,26 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) { } func createAccountAdapter( - marshalizer marshal.Marshalizer, + marshaller marshal.Marshalizer, hasher hashing.Hasher, accountFactory state.AccountFactory, trieStorage common.StorageManager, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, 5) + tr, err := trie.NewTrie(trieStorage, marshaller, hasher, 5) if err != nil { return nil, err } - adb, err := state.NewAccountsDB(tr, hasher, marshalizer, accountFactory, disabled.NewDisabledStoragePruningManager(), common.Normal) + args := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, err := state.NewAccountsDB(args) if err != nil { return nil, err } diff --git a/factory/stateComponents.go b/factory/stateComponents.go index a0165eb89e7..c89ba401380 100644 --- a/factory/stateComponents.go +++ b/factory/stateComponents.go @@ -125,26 +125,30 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. return nil, nil, err } - accountsAdapter, err := state.NewAccountsDB( - merkleTrie, - scf.core.Hasher(), - scf.core.InternalMarshalizer(), - accountFactory, - storagePruning, - scf.processingMode, - ) + argsProcessingAccountsDB := state.ArgsAccountsDB{ + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + ProcessingMode: scf.processingMode, + ProcessStatusHandler: scf.core.ProcessStatusHandler(), + } + accountsAdapter, err := state.NewAccountsDB(argsProcessingAccountsDB) if err != nil { return nil, nil, fmt.Errorf("%w: %s", errors.ErrAccountsAdapterCreation, err.Error()) } - accountsAdapterAPI, err := state.NewAccountsDB( - merkleTrie, - scf.core.Hasher(), - scf.core.InternalMarshalizer(), - accountFactory, - storagePruning, - scf.processingMode, - ) + argsAPIAccountsDB := state.ArgsAccountsDB{ + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + ProcessingMode: scf.processingMode, + ProcessStatusHandler: scf.core.ProcessStatusHandler(), + } + accountsAdapterAPI, err := state.NewAccountsDB(argsAPIAccountsDB) if err != nil { return nil, nil, fmt.Errorf("accounts adapter API: %w: %s", errors.ErrAccountsAdapterCreation, err.Error()) } @@ -165,13 +169,16 @@ func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.Tries return nil, err } - peerAdapter, err := state.NewPeerAccountsDB( - merkleTrie, - scf.core.Hasher(), - scf.core.InternalMarshalizer(), - accountFactory, - storagePruning, - ) + argsProcessingPeerAccountsDB := state.ArgsAccountsDB{ + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + ProcessingMode: scf.processingMode, + ProcessStatusHandler: scf.core.ProcessStatusHandler(), + } + peerAdapter, err := state.NewPeerAccountsDB(argsProcessingPeerAccountsDB) if err != nil { return nil, err } diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index 5102be8fb6c..12aaf62ff70 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -4,6 +4,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" + commonDisabled "github.com/ElrondNetwork/elrond-go/common/disabled" "github.com/ElrondNetwork/elrond-go/state" "github.com/ElrondNetwork/elrond-go/state/storagePruningManager/disabled" "github.com/ElrondNetwork/elrond-go/trie" @@ -12,24 +13,27 @@ import ( const maxTrieLevelInMemory = uint(5) func createAccountAdapter( - marshalizer marshal.Marshalizer, + marshaller marshal.Marshalizer, hasher hashing.Hasher, accountFactory state.AccountFactory, trieStorage common.StorageManager, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, maxTrieLevelInMemory) + tr, err := trie.NewTrie(trieStorage, marshaller, hasher, maxTrieLevelInMemory) if err != nil { return nil, err } - adb, err := state.NewAccountsDB( - tr, - hasher, - marshalizer, - accountFactory, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + args := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + ProcessingMode: common.Normal, + ProcessStatusHandler: &commonDisabled.ProcessStatusHandler{}, + } + + adb, err := state.NewAccountsDB(args) if err != nil { return nil, err } diff --git a/integrationTests/consensus/testInitializer.go b/integrationTests/consensus/testInitializer.go index c05908909ed..077d20415bb 100644 --- a/integrationTests/consensus/testInitializer.go +++ b/integrationTests/consensus/testInitializer.go @@ -157,7 +157,7 @@ func createTestStore() dataRetriever.StorageService { return store } -func createAccountsDB(marshalizer marshal.Marshalizer) state.AccountsAdapter { +func createAccountsDB(marshaller marshal.Marshalizer) state.AccountsAdapter { marsh := &marshal.GogoProtoMarshalizer{} hasher := sha256.NewSha256() store := createMemUnit() @@ -183,7 +183,7 @@ func createAccountsDB(marshalizer marshal.Marshalizer) state.AccountsAdapter { DB: store, MainStorer: createMemUnit(), CheckpointsStorer: createMemUnit(), - Marshalizer: marshalizer, + Marshalizer: marshaller, Hasher: hasher, SnapshotDbConfig: cfg, GeneralConfig: generalCfg, @@ -199,18 +199,22 @@ func createAccountsDB(marshalizer marshal.Marshalizer) state.AccountsAdapter { ewl, generalCfg.PruningBufferLen, ) - adb, _ := state.NewAccountsDB( - tr, - sha256.NewSha256(), - marshalizer, - &mock.AccountsFactoryStub{ + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: sha256.NewSha256(), + Marshaller: marshaller, + AccountFactory: &mock.AccountsFactoryStub{ CreateAccountCalled: func(address []byte) (wrapper vmcommon.AccountHandler, e error) { return state.NewUserAccount(address) }, }, - storagePruning, - common.Normal, - ) + StoragePruningManager: storagePruning, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + + adb, _ := state.NewAccountsDB(argsAccountsDB) return adb } diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index 5b59a890f70..9db76f78cf6 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -1060,7 +1060,17 @@ func createAccounts( maxTrieLevelInMemory := uint(5) tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) - adb, _ := state.NewAccountsDB(tr, integrationTests.TestHasher, integrationTests.TestMarshalizer, factory.NewAccountCreator(), spm, common.Normal) + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) addr := make([][]byte, nrOfAccounts) for i := 0; i < nrOfAccounts; i++ { @@ -2369,7 +2379,17 @@ func createAccountsDBTestSetup() *state.AccountsDB { maxTrieLevelInMemory := uint(5) tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) - adb, _ := state.NewAccountsDB(tr, integrationTests.TestHasher, integrationTests.TestMarshalizer, factory.NewAccountCreator(), spm, common.Normal) + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) return adb } diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index 1b8141cbf8f..b2b4a466e89 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -498,7 +498,16 @@ func CreateAccountsDB( ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, memorydb.New(), TestMarshalizer) accountFactory := getAccountFactory(accountType) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) - adb, _ := state.NewAccountsDB(tr, sha256.NewSha256(), TestMarshalizer, accountFactory, spm, common.Normal) + args := state.ArgsAccountsDB{ + Trie: tr, + Hasher: sha256.NewSha256(), + Marshaller: TestMarshalizer, + AccountFactory: accountFactory, + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(args) return adb, tr } diff --git a/integrationTests/vm/testInitializer.go b/integrationTests/vm/testInitializer.go index aa887268807..311ee1f39d0 100644 --- a/integrationTests/vm/testInitializer.go +++ b/integrationTests/vm/testInitializer.go @@ -285,9 +285,9 @@ func CreateMemUnit() storage.Storer { // CreateInMemoryShardAccountsDB - func CreateInMemoryShardAccountsDB() *state.AccountsDB { - marsh := &marshal.GogoProtoMarshalizer{} + marshaller := &marshal.GogoProtoMarshalizer{} store := CreateMemUnit() - ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, memorydb.New(), marsh) + ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, memorydb.New(), marshaller) generalCfg := config.TrieStorageManagerConfig{ PruningBufferLen: 1000, SnapshotsBufferLen: 10, @@ -298,7 +298,7 @@ func CreateInMemoryShardAccountsDB() *state.AccountsDB { DB: store, MainStorer: CreateMemUnit(), CheckpointsStorer: CreateMemUnit(), - Marshalizer: marsh, + Marshalizer: marshaller, Hasher: testHasher, SnapshotDbConfig: config.DBConfig{ FilePath: "TrieStorage", @@ -314,9 +314,19 @@ func CreateInMemoryShardAccountsDB() *state.AccountsDB { } trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marsh, testHasher, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorage, marshaller, testHasher, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) - adb, _ := state.NewAccountsDB(tr, testHasher, marsh, &accountFactory{}, spm, common.Normal) + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: testHasher, + Marshaller: marshaller, + AccountFactory: &accountFactory{}, + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) return adb } diff --git a/state/accountsDB.go b/state/accountsDB.go index af3ff5220f4..72829782ee3 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -71,7 +71,7 @@ type snapshotInfo struct { type AccountsDB struct { mainTrie common.Trie hasher hashing.Hasher - marshalizer marshal.Marshalizer + marshaller marshal.Marshalizer accountFactory AccountFactory storagePruningManager StoragePruningManager obsoleteDataTrieHashes map[string][][]byte @@ -85,45 +85,39 @@ type AccountsDB struct { processingMode common.NodeProcessingMode numCheckpoints uint32 loadCodeMeasurements *loadingMeasurements + processStatusHandler common.ProcessStatusHandler stackDebug []byte } var log = logger.GetOrCreate("state") +// ArgsAccountsDB is the arguments DTO for the AccountsDB instance +type ArgsAccountsDB struct { + Trie common.Trie + Hasher hashing.Hasher + Marshaller marshal.Marshalizer + AccountFactory AccountFactory + StoragePruningManager StoragePruningManager + ProcessingMode common.NodeProcessingMode + ProcessStatusHandler common.ProcessStatusHandler +} + // NewAccountsDB creates a new account manager -func NewAccountsDB( - trie common.Trie, - hasher hashing.Hasher, - marshalizer marshal.Marshalizer, - accountFactory AccountFactory, - storagePruningManager StoragePruningManager, - processingMode common.NodeProcessingMode, -) (*AccountsDB, error) { - if check.IfNil(trie) { - return nil, ErrNilTrie - } - if check.IfNil(hasher) { - return nil, ErrNilHasher - } - if check.IfNil(marshalizer) { - return nil, ErrNilMarshalizer - } - if check.IfNil(accountFactory) { - return nil, ErrNilAccountFactory - } - if check.IfNil(storagePruningManager) { - return nil, ErrNilStoragePruningManager +func NewAccountsDB(args ArgsAccountsDB) (*AccountsDB, error) { + err := checkArgsAccountsDB(args) + if err != nil { + return nil, err } - trieStorageManager := trie.GetStorageManager() + trieStorageManager := args.Trie.GetStorageManager() numCheckpoints := getNumCheckpoints(trieStorageManager) adb := &AccountsDB{ - mainTrie: trie, - hasher: hasher, - marshalizer: marshalizer, - accountFactory: accountFactory, - storagePruningManager: storagePruningManager, + mainTrie: args.Trie, + hasher: args.Hasher, + marshaller: args.Marshaller, + accountFactory: args.AccountFactory, + storagePruningManager: args.StoragePruningManager, entries: make([]JournalEntry, 0), mutOp: sync.RWMutex{}, dataTries: NewDataTriesHolder(), @@ -132,8 +126,9 @@ func NewAccountsDB( loadCodeMeasurements: &loadingMeasurements{ identifier: "load code", }, - processingMode: processingMode, - lastSnapshot: &snapshotInfo{}, + processingMode: args.ProcessingMode, + lastSnapshot: &snapshotInfo{}, + processStatusHandler: args.ProcessStatusHandler, } val, err := trieStorageManager.GetFromCurrentEpoch([]byte(common.ActiveDBKey)) @@ -144,6 +139,29 @@ func NewAccountsDB( return adb, nil } +func checkArgsAccountsDB(args ArgsAccountsDB) error { + if check.IfNil(args.Trie) { + return ErrNilTrie + } + if check.IfNil(args.Hasher) { + return ErrNilHasher + } + if check.IfNil(args.Marshaller) { + return ErrNilMarshalizer + } + if check.IfNil(args.AccountFactory) { + return ErrNilAccountFactory + } + if check.IfNil(args.StoragePruningManager) { + return ErrNilStoragePruningManager + } + if check.IfNil(args.ProcessStatusHandler) { + return ErrNilProcessStatusHandler + } + + return nil +} + func startSnapshotAfterRestart(adb AccountsAdapter, tsm common.StorageManager) { rootHash, err := tsm.Get([]byte(lastSnapshotStarted)) if err != nil { @@ -213,7 +231,7 @@ func (adb *AccountsDB) GetCode(codeHash []byte) []byte { return nil } - err = adb.marshalizer.Unmarshal(&codeEntry, val) + err = adb.marshaller.Unmarshal(&codeEntry, val) if err != nil { return nil } @@ -324,7 +342,7 @@ func (adb *AccountsDB) saveCode(newAcc, oldAcc baseAccountHandler) error { return err } - entry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, newCodeHash, adb.mainTrie, adb.marshalizer) + entry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, newCodeHash, adb.mainTrie, adb.marshaller) if err != nil { return err } @@ -335,7 +353,7 @@ func (adb *AccountsDB) saveCode(newAcc, oldAcc baseAccountHandler) error { } func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error) { - oldCodeEntry, err := getCodeEntry(oldCodeHash, adb.mainTrie, adb.marshalizer) + oldCodeEntry, err := getCodeEntry(oldCodeHash, adb.mainTrie, adb.marshaller) if err != nil { return nil, err } @@ -359,7 +377,7 @@ func (adb *AccountsDB) updateOldCodeEntry(oldCodeHash []byte) (*CodeEntry, error } oldCodeEntry.NumReferences-- - err = saveCodeEntry(oldCodeHash, oldCodeEntry, adb.mainTrie, adb.marshalizer) + err = saveCodeEntry(oldCodeHash, oldCodeEntry, adb.mainTrie, adb.marshaller) if err != nil { return nil, err } @@ -372,7 +390,7 @@ func (adb *AccountsDB) updateNewCodeEntry(newCodeHash []byte, newCode []byte) er return nil } - newCodeEntry, err := getCodeEntry(newCodeHash, adb.mainTrie, adb.marshalizer) + newCodeEntry, err := getCodeEntry(newCodeHash, adb.mainTrie, adb.marshaller) if err != nil { return err } @@ -384,7 +402,7 @@ func (adb *AccountsDB) updateNewCodeEntry(newCodeHash []byte, newCode []byte) er } newCodeEntry.NumReferences++ - err = saveCodeEntry(newCodeHash, newCodeEntry, adb.mainTrie, adb.marshalizer) + err = saveCodeEntry(newCodeHash, newCodeEntry, adb.mainTrie, adb.marshaller) if err != nil { return err } @@ -522,8 +540,8 @@ func (adb *AccountsDB) saveAccountToTrie(accountHandler vmcommon.AccountHandler) "address", hex.EncodeToString(accountHandler.AddressBytes()), ) - // pass the reference to marshalizer, otherwise it will fail marshalizing balance - buff, err := adb.marshalizer.Marshal(accountHandler) + // pass the reference to marshaller, otherwise it will fail marshalling balance + buff, err := adb.marshaller.Marshal(accountHandler) if err != nil { return err } @@ -620,7 +638,7 @@ func (adb *AccountsDB) removeCode(baseAcc baseAccountHandler) error { return err } - codeChangeEntry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, nil, adb.mainTrie, adb.marshalizer) + codeChangeEntry, err := NewJournalEntryCode(unmodifiedOldCodeEntry, oldCodeHash, nil, adb.mainTrie, adb.marshaller) if err != nil { return err } @@ -675,7 +693,7 @@ func (adb *AccountsDB) getAccount(address []byte) (vmcommon.AccountHandler, erro return nil, err } - err = adb.marshalizer.Unmarshal(acnt, val) + err = adb.marshaller.Unmarshal(acnt, val) if err != nil { return nil, err } @@ -729,7 +747,7 @@ func (adb *AccountsDB) GetAccountFromBytes(address []byte, accountBytes []byte) return nil, err } - err = adb.marshalizer.Unmarshal(acnt, accountBytes) + err = adb.marshaller.Unmarshal(acnt, accountBytes) if err != nil { return nil, err } @@ -759,7 +777,7 @@ func (adb *AccountsDB) loadCode(accountHandler baseAccountHandler) error { } var codeEntry CodeEntry - err = adb.marshalizer.Unmarshal(&codeEntry, val) + err = adb.marshaller.Unmarshal(&codeEntry, val) if err != nil { return err } @@ -1002,7 +1020,7 @@ func (adb *AccountsDB) RecreateAllTries(rootHash []byte) (map[string]common.Trie for leaf := range leavesChannel { account := &userAccount{} - err = adb.marshalizer.Unmarshal(account, leaf.Value()) + err = adb.marshaller.Unmarshal(account, leaf.Value()) if err != nil { log.Trace("this must be a leaf with code", "err", err) continue @@ -1123,9 +1141,7 @@ func (adb *AccountsDB) SnapshotState(rootHash []byte) { handleLoggingWhenError("error while putting active DB value into main storer", errPut) }() - if adb.processingMode == common.ImportDb { - stats.WaitForSnapshotsToFinish() - } + adb.waitForCompletionIfRunningInImportDB(stats) } func printStats(stats *snapshotStatistics, identifier string, rootHash []byte) { @@ -1153,7 +1169,7 @@ func (adb *AccountsDB) snapshotUserAccountDataTrie( ) { for leaf := range leavesChannel { account := &userAccount{} - err := adb.marshalizer.Unmarshal(account, leaf.Value()) + err := adb.marshaller.Unmarshal(account, leaf.Value()) if err != nil { log.Trace("this must be a leaf with code", "err", err) continue @@ -1202,9 +1218,18 @@ func (adb *AccountsDB) setStateCheckpoint(rootHash []byte) { go printStats(stats, "setStateCheckpoint user trie", rootHash) - if adb.processingMode == common.ImportDb { - stats.WaitForSnapshotsToFinish() + adb.waitForCompletionIfRunningInImportDB(stats) +} + +func (adb *AccountsDB) waitForCompletionIfRunningInImportDB(stats common.SnapshotStatisticsHandler) { + if adb.processingMode != common.ImportDb { + return } + + log.Debug("manually setting idle on the process status handler in order to be able to start & complete the snapshotting/checkpointing process") + adb.processStatusHandler.SetIdle() + + stats.WaitForSnapshotsToFinish() } func (adb *AccountsDB) increaseNumCheckpoints() { diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index e823f31b8ee..c5e0f0eaafd 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -36,19 +36,31 @@ import ( const trieDbOperationDelay = time.Second -func generateAccountDBFromTrie(trie common.Trie) *state.AccountsDB { - accnt, _ := state.NewAccountsDB( - trie, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{ +func createMockAccountsDBArgs() state.ArgsAccountsDB { + return state.ArgsAccountsDB{ + Trie: &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{} + }, + }, + Hasher: &hashingMocks.HasherMock{}, + Marshaller: &testscommon.MarshalizerMock{}, + AccountFactory: &stateMock.AccountsFactoryStub{ CreateAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { return stateMock.NewAccountWrapMock(address), nil }, }, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } +} + +func generateAccountDBFromTrie(trie common.Trie) *state.AccountsDB { + args := createMockAccountsDBArgs() + args.Trie = trie + + accnt, _ := state.NewAccountsDB(args) return accnt } @@ -80,15 +92,15 @@ func getDefaultStateComponents( MaxSnapshots: 2, SnapshotsGoroutineNum: 1, } - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} args := trie.NewTrieStorageManagerArgs{ DB: testscommon.NewMemDbMock(), MainStorer: testscommon.NewSnapshotPruningStorerMock(), CheckpointsStorer: testscommon.NewSnapshotPruningStorerMock(), - Marshalizer: marshalizer, - Hasher: hsh, + Marshalizer: marshaller, + Hasher: hasher, SnapshotDbConfig: config.DBConfig{ Type: "MemoryDB", }, @@ -98,114 +110,106 @@ func getDefaultStateComponents( IdleProvider: &testscommon.ProcessStatusHandlerStub{}, } trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshalizer, hsh, 5) - ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshalizer) + tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, 5) + ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshaller) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, generalCfg.PruningBufferLen) - adb, _ := state.NewAccountsDB(tr, hsh, marshalizer, factory.NewAccountCreator(), spm, common.Normal) + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) return adb, tr, trieStorage } -// ------- NewAccountsDB - -func TestNewAccountsDB_WithNilTrieShouldErr(t *testing.T) { +func TestNewAccountsDB(t *testing.T) { t.Parallel() - adb, err := state.NewAccountsDB( - nil, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + t.Run("nil trie should error", func(t *testing.T) { + t.Parallel() - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilTrie, err) -} + args := createMockAccountsDBArgs() + args.Trie = nil -func TestNewAccountsDB_WithNilHasherShouldErr(t *testing.T) { - t.Parallel() + adb, err := state.NewAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilTrie, err) + }) + t.Run("nil hasher should error", func(t *testing.T) { + t.Parallel() - adb, err := state.NewAccountsDB( - &trieMock.TrieStub{}, - nil, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + args := createMockAccountsDBArgs() + args.Hasher = nil - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilHasher, err) -} + adb, err := state.NewAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilHasher, err) + }) + t.Run("nil hasher should error", func(t *testing.T) { + t.Parallel() -func TestNewAccountsDB_WithNilMarshalizerShouldErr(t *testing.T) { - t.Parallel() + args := createMockAccountsDBArgs() + args.Hasher = nil - adb, err := state.NewAccountsDB( - &trieMock.TrieStub{}, - &hashingMocks.HasherMock{}, - nil, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + adb, err := state.NewAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilHasher, err) + }) + t.Run("nil marshaller should error", func(t *testing.T) { + t.Parallel() - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilMarshalizer, err) -} + args := createMockAccountsDBArgs() + args.Marshaller = nil -func TestNewAccountsDB_WithNilAddressFactoryShouldErr(t *testing.T) { - t.Parallel() + adb, err := state.NewAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilMarshalizer, err) + }) + t.Run("nil account factory should error", func(t *testing.T) { + t.Parallel() - adb, err := state.NewAccountsDB( - &trieMock.TrieStub{}, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - nil, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + args := createMockAccountsDBArgs() + args.AccountFactory = nil - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilAccountFactory, err) -} + adb, err := state.NewAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilAccountFactory, err) + }) + t.Run("nil storage pruning manager should error", func(t *testing.T) { + t.Parallel() -func TestNewAccountsDB_WithNilStoragePruningManagerShouldErr(t *testing.T) { - t.Parallel() + args := createMockAccountsDBArgs() + args.StoragePruningManager = nil - adb, err := state.NewAccountsDB( - &trieMock.TrieStub{}, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - nil, - common.Normal, - ) + adb, err := state.NewAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilStoragePruningManager, err) + }) + t.Run("nil process status handler should error", func(t *testing.T) { + t.Parallel() - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilStoragePruningManager, err) -} + args := createMockAccountsDBArgs() + args.ProcessStatusHandler = nil -func TestNewAccountsDB_OkValsShouldWork(t *testing.T) { - t.Parallel() + adb, err := state.NewAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilProcessStatusHandler, err) + }) + t.Run("should work", func(t *testing.T) { + t.Parallel() - adb, err := state.NewAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{} - }, - }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + args := createMockAccountsDBArgs() - assert.Nil(t, err) - assert.False(t, check.IfNil(adb)) + adb, err := state.NewAccountsDB(args) + assert.False(t, check.IfNil(adb)) + assert.Nil(t, err) + }) } func TestNewAccountsDB_SetsNumCheckpoints(t *testing.T) { @@ -219,22 +223,18 @@ func TestNewAccountsDB_SetsNumCheckpoints(t *testing.T) { binary.BigEndian.PutUint32(numCheckpointsVal, numCheckpoints) _ = db.Put(numCheckpointsKey, numCheckpointsVal) - adb, _ := state.NewAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{ - GetCalled: func(key []byte) ([]byte, error) { - return db.Get(key) - }, - } - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{ + GetCalled: func(key []byte) ([]byte, error) { + return db.Get(key) + }, + } }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + } + + adb, _ := state.NewAccountsDB(args) assert.Equal(t, numCheckpoints, adb.GetNumCheckpoints()) } @@ -358,7 +358,7 @@ func TestAccountsDB_SaveAccountSavesCodeAndDataTrieForUserAccount(t *testing.T) assert.NotNil(t, acc.GetRootHash()) } -func TestAccountsDB_SaveAccountMalfunctionMarshalizerShouldErr(t *testing.T) { +func TestAccountsDB_SaveAccountMalfunctionMarshallerShouldErr(t *testing.T) { t.Parallel() account := generateAccount() @@ -367,21 +367,14 @@ func TestAccountsDB_SaveAccountMalfunctionMarshalizerShouldErr(t *testing.T) { return &testscommon.StorageManagerStub{} }, } - marshalizer := &testscommon.MarshalizerMock{} - adb, _ := state.NewAccountsDB( - mockTrie, - &hashingMocks.HasherMock{}, - marshalizer, - &stateMock.AccountsFactoryStub{ - CreateAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { - return stateMock.NewAccountWrapMock(address), nil - }, - }, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + marshaller := &testscommon.MarshalizerMock{} + args := createMockAccountsDBArgs() + args.Marshaller = marshaller + args.Trie = mockTrie - marshalizer.Fail = true + adb, _ := state.NewAccountsDB(args) + + marshaller.Fail = true // should return error err := adb.SaveAccount(account) @@ -416,10 +409,10 @@ func TestAccountsDB_RemoveAccountShouldWork(t *testing.T) { t.Parallel() wasCalled := false - marshalizer := &testscommon.MarshalizerMock{} + marshaller := &testscommon.MarshalizerMock{} trieStub := &trieMock.TrieStub{ GetCalled: func(key []byte) (i []byte, err error) { - return marshalizer.Marshal(stateMock.AccountWrapMock{}) + return marshaller.Marshal(stateMock.AccountWrapMock{}) }, UpdateCalled: func(key, value []byte) error { wasCalled = true @@ -490,15 +483,15 @@ func TestAccountsDB_LoadAccountExistingShouldLoadDataTrie(t *testing.T) { acc.SetCodeHash(codeHash) code := []byte("code") dataTrie := &trieMock.TrieStub{} - marshalizer := &testscommon.MarshalizerMock{} + marshaller := &testscommon.MarshalizerMock{} trieStub := &trieMock.TrieStub{ GetCalled: func(key []byte) (i []byte, e error) { if bytes.Equal(key, acc.AddressBytes()) { - return marshalizer.Marshal(acc) + return marshaller.Marshal(acc) } if bytes.Equal(key, codeHash) { - return marshalizer.Marshal(state.CodeEntry{Code: code}) + return marshaller.Marshal(state.CodeEntry{Code: code}) } return nil, nil }, @@ -566,15 +559,15 @@ func TestAccountsDB_GetExistingAccountFoundShouldRetAccount(t *testing.T) { acc.SetCodeHash(codeHash) code := []byte("code") dataTrie := &trieMock.TrieStub{} - marshalizer := &testscommon.MarshalizerMock{} + marshaller := &testscommon.MarshalizerMock{} trieStub := &trieMock.TrieStub{ GetCalled: func(key []byte) (i []byte, e error) { if bytes.Equal(key, acc.AddressBytes()) { - return marshalizer.Marshal(acc) + return marshaller.Marshal(acc) } if bytes.Equal(key, codeHash) { - return marshalizer.Marshal(state.CodeEntry{Code: code}) + return marshaller.Marshal(state.CodeEntry{Code: code}) } return nil, nil }, @@ -611,8 +604,8 @@ func TestAccountsDB_GetAccountAccountNotFound(t *testing.T) { testAccount.MockValue = 45 // Step 2. marshalize the DbAccount - marshalizer := testscommon.MarshalizerMock{} - buff, err := marshalizer.Marshal(testAccount) + marshaller := &testscommon.MarshalizerMock{} + buff, err := marshaller.Marshal(testAccount) assert.Nil(t, err) tr.GetCalled = func(key []byte) (bytes []byte, e error) { @@ -620,18 +613,11 @@ func TestAccountsDB_GetAccountAccountNotFound(t *testing.T) { return buff, nil } - adb, _ := state.NewAccountsDB( - tr, - &hashingMocks.HasherMock{}, - &marshalizer, - &stateMock.AccountsFactoryStub{ - CreateAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { - return stateMock.NewAccountWrapMock(address), nil - }, - }, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + args := createMockAccountsDBArgs() + args.Trie = tr + args.Marshaller = marshaller + + adb, _ := state.NewAccountsDB(args) // Step 3. call get, should return a copy of DbAccount, recover an Account object recoveredAccount, err := adb.GetAccount(adr) @@ -687,30 +673,23 @@ func TestAccountsDB_LoadCodeOkValsShouldWork(t *testing.T) { }, } adr, account, _ := generateAddressAccountAccountsDB(tr) - marshalizer := testscommon.MarshalizerMock{} + marshaller := &testscommon.MarshalizerMock{} - trieStub := trieMock.TrieStub{ + trieStub := &trieMock.TrieStub{ GetCalled: func(key []byte) (bytes []byte, e error) { // will return adr.Bytes() so its hash will correspond to adr.Hash() - return marshalizer.Marshal(&state.CodeEntry{Code: adr}) + return marshaller.Marshal(&state.CodeEntry{Code: adr}) }, GetStorageManagerCalled: func() common.StorageManager { return &testscommon.StorageManagerStub{} }, } - adb, _ := state.NewAccountsDB( - &trieStub, - &hashingMocks.HasherMock{}, - &marshalizer, - &stateMock.AccountsFactoryStub{ - CreateAccountCalled: func(address []byte) (vmcommon.AccountHandler, error) { - return stateMock.NewAccountWrapMock(address), nil - }, - }, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + args := createMockAccountsDBArgs() + args.Trie = trieStub + args.Marshaller = marshaller + + adb, _ := state.NewAccountsDB(args) // just search a hash. Any hash will do account.SetCodeHash(adr) @@ -845,8 +824,8 @@ func TestAccountsDB_CommitShouldCallCommitFromTrie(t *testing.T) { t.Parallel() commitCalled := 0 - marshalizer := &testscommon.MarshalizerMock{} - serializedAccount, _ := marshalizer.Marshal(stateMock.AccountWrapMock{}) + marshaller := &testscommon.MarshalizerMock{} + serializedAccount, _ := marshaller.Marshal(stateMock.AccountWrapMock{}) trieStub := trieMock.TrieStub{ CommitCalled: func() error { commitCalled++ @@ -1328,7 +1307,7 @@ func checkCodeEntry( codeHash []byte, expectedCode []byte, expectedNumReferences uint32, - marshalizer marshal.Marshalizer, + marshaller marshal.Marshalizer, tr common.Trie, t *testing.T, ) { @@ -1337,7 +1316,7 @@ func checkCodeEntry( assert.NotNil(t, val) var codeEntry state.CodeEntry - err = marshalizer.Unmarshal(&codeEntry, val) + err = marshaller.Unmarshal(&codeEntry, val) assert.Nil(t, err) assert.Equal(t, expectedCode, codeEntry.Code) @@ -1347,8 +1326,8 @@ func checkCodeEntry( func TestAccountsDB_SaveAccountSavesCodeIfCodeHashIsSet(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} tr, adb := getDefaultTrieAndAccountsDb() addr := make([]byte, 32) @@ -1357,12 +1336,12 @@ func TestAccountsDB_SaveAccountSavesCodeIfCodeHashIsSet(t *testing.T) { code := []byte("code") userAcc.SetCode(code) - codeHash := hsh.Compute(string(code)) + codeHash := hasher.Compute(string(code)) userAcc.SetCodeHash(codeHash) _ = adb.SaveAccount(acc) - checkCodeEntry(codeHash, code, 1, marshalizer, tr, t) + checkCodeEntry(codeHash, code, 1, marshaller, tr, t) } func TestAccountsDB_saveCode_OldCodeAndNewCodeAreNil(t *testing.T) { @@ -1382,8 +1361,8 @@ func TestAccountsDB_saveCode_OldCodeAndNewCodeAreNil(t *testing.T) { func TestAccountsDB_saveCode_OldCodeIsNilAndNewCodeIsNotNilAndRevert(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} tr, adb := getDefaultTrieAndAccountsDb() addr := make([]byte, 32) @@ -1392,13 +1371,13 @@ func TestAccountsDB_saveCode_OldCodeIsNilAndNewCodeIsNotNilAndRevert(t *testing. code := []byte("code") userAcc.SetCode(code) - expectedCodeHash := hsh.Compute(string(code)) + expectedCodeHash := hasher.Compute(string(code)) err := adb.SaveAccount(userAcc) assert.Nil(t, err) assert.Equal(t, expectedCodeHash, userAcc.GetCodeHash()) - checkCodeEntry(userAcc.GetCodeHash(), code, 1, marshalizer, tr, t) + checkCodeEntry(userAcc.GetCodeHash(), code, 1, marshaller, tr, t) err = adb.RevertToSnapshot(1) assert.Nil(t, err) @@ -1411,8 +1390,8 @@ func TestAccountsDB_saveCode_OldCodeIsNilAndNewCodeIsNotNilAndRevert(t *testing. func TestAccountsDB_saveCode_OldCodeIsNilAndNewCodeAlreadyExistsAndRevert(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} tr, adb := getDefaultTrieAndAccountsDb() addr := make([]byte, 32) @@ -1421,7 +1400,7 @@ func TestAccountsDB_saveCode_OldCodeIsNilAndNewCodeAlreadyExistsAndRevert(t *tes code := []byte("code") userAcc.SetCode(code) - expectedCodeHash := hsh.Compute(string(code)) + expectedCodeHash := hasher.Compute(string(code)) _ = adb.SaveAccount(userAcc) journalLen := adb.JournalLen() @@ -1436,19 +1415,19 @@ func TestAccountsDB_saveCode_OldCodeIsNilAndNewCodeAlreadyExistsAndRevert(t *tes assert.Nil(t, err) assert.Equal(t, expectedCodeHash, userAcc.GetCodeHash()) - checkCodeEntry(userAcc.GetCodeHash(), code, 2, marshalizer, tr, t) + checkCodeEntry(userAcc.GetCodeHash(), code, 2, marshaller, tr, t) err = adb.RevertToSnapshot(journalLen) assert.Nil(t, err) - checkCodeEntry(expectedCodeHash, code, 1, marshalizer, tr, t) + checkCodeEntry(expectedCodeHash, code, 1, marshaller, tr, t) } func TestAccountsDB_saveCode_OldCodeExistsAndNewCodeIsNilAndRevert(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} tr, adb := getDefaultTrieAndAccountsDb() addr := make([]byte, 32) @@ -1457,7 +1436,7 @@ func TestAccountsDB_saveCode_OldCodeExistsAndNewCodeIsNilAndRevert(t *testing.T) code := []byte("code") userAcc.SetCode(code) - oldCodeHash := hsh.Compute(string(code)) + oldCodeHash := hasher.Compute(string(code)) _ = adb.SaveAccount(userAcc) journalLen := adb.JournalLen() @@ -1477,14 +1456,14 @@ func TestAccountsDB_saveCode_OldCodeExistsAndNewCodeIsNilAndRevert(t *testing.T) err = adb.RevertToSnapshot(journalLen) assert.Nil(t, err) - checkCodeEntry(oldCodeHash, code, 1, marshalizer, tr, t) + checkCodeEntry(oldCodeHash, code, 1, marshaller, tr, t) } func TestAccountsDB_saveCode_OldCodeExistsAndNewCodeExistsAndRevert(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} tr, adb := getDefaultTrieAndAccountsDb() addr := make([]byte, 32) @@ -1492,7 +1471,7 @@ func TestAccountsDB_saveCode_OldCodeExistsAndNewCodeExistsAndRevert(t *testing.T userAcc := acc.(state.UserAccountHandler) oldCode := []byte("code1") userAcc.SetCode(oldCode) - oldCodeHash := hsh.Compute(string(oldCode)) + oldCodeHash := hasher.Compute(string(oldCode)) _ = adb.SaveAccount(userAcc) addr1 := make([]byte, 32) @@ -1501,7 +1480,7 @@ func TestAccountsDB_saveCode_OldCodeExistsAndNewCodeExistsAndRevert(t *testing.T userAcc = acc.(state.UserAccountHandler) newCode := []byte("code2") userAcc.SetCode(newCode) - newCodeHash := hsh.Compute(string(newCode)) + newCodeHash := hasher.Compute(string(newCode)) _ = adb.SaveAccount(userAcc) journalLen := adb.JournalLen() @@ -1517,20 +1496,20 @@ func TestAccountsDB_saveCode_OldCodeExistsAndNewCodeExistsAndRevert(t *testing.T assert.Nil(t, err) assert.Nil(t, val) - checkCodeEntry(newCodeHash, newCode, 2, marshalizer, tr, t) + checkCodeEntry(newCodeHash, newCode, 2, marshaller, tr, t) err = adb.RevertToSnapshot(journalLen) assert.Nil(t, err) - checkCodeEntry(oldCodeHash, oldCode, 1, marshalizer, tr, t) - checkCodeEntry(newCodeHash, newCode, 1, marshalizer, tr, t) + checkCodeEntry(oldCodeHash, oldCode, 1, marshaller, tr, t) + checkCodeEntry(newCodeHash, newCode, 1, marshaller, tr, t) } func TestAccountsDB_saveCode_OldCodeIsReferencedMultipleTimesAndNewCodeIsNilAndRevert(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} tr, adb := getDefaultTrieAndAccountsDb() addr := make([]byte, 32) @@ -1539,7 +1518,7 @@ func TestAccountsDB_saveCode_OldCodeIsReferencedMultipleTimesAndNewCodeIsNilAndR code := []byte("code") userAcc.SetCode(code) - oldCodeHash := hsh.Compute(string(code)) + oldCodeHash := hasher.Compute(string(code)) _ = adb.SaveAccount(userAcc) addr1 := make([]byte, 32) @@ -1558,18 +1537,18 @@ func TestAccountsDB_saveCode_OldCodeIsReferencedMultipleTimesAndNewCodeIsNilAndR assert.Nil(t, err) assert.Nil(t, userAcc.GetCodeHash()) - checkCodeEntry(oldCodeHash, code, 1, marshalizer, tr, t) + checkCodeEntry(oldCodeHash, code, 1, marshaller, tr, t) err = adb.RevertToSnapshot(journalLen) assert.Nil(t, err) - checkCodeEntry(oldCodeHash, code, 2, marshalizer, tr, t) + checkCodeEntry(oldCodeHash, code, 2, marshaller, tr, t) } func TestAccountsDB_RemoveAccountAlsoRemovesCodeAndRevertsCorrectly(t *testing.T) { t.Parallel() - hsh := &hashingMocks.HasherMock{} + hasher := &hashingMocks.HasherMock{} tr, adb := getDefaultTrieAndAccountsDb() addr := make([]byte, 32) @@ -1578,7 +1557,7 @@ func TestAccountsDB_RemoveAccountAlsoRemovesCodeAndRevertsCorrectly(t *testing.T code := []byte("code") userAcc.SetCode(code) - oldCodeHash := hsh.Compute(string(code)) + oldCodeHash := hasher.Compute(string(code)) _ = adb.SaveAccount(userAcc) snapshot := adb.JournalLen() @@ -1604,15 +1583,15 @@ func TestAccountsDB_RemoveAccountAlsoRemovesCodeAndRevertsCorrectly(t *testing.T func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} - ewl := stateMock.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshalizer) + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} + ewl := stateMock.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshaller) args := trie.NewTrieStorageManagerArgs{ DB: testscommon.NewMemDbMock(), MainStorer: testscommon.CreateMemUnit(), CheckpointsStorer: testscommon.CreateMemUnit(), - Marshalizer: marshalizer, - Hasher: hsh, + Marshalizer: marshaller, + Hasher: hasher, SnapshotDbConfig: config.DBConfig{}, GeneralConfig: config.TrieStorageManagerConfig{SnapshotsGoroutineNum: 1}, CheckpointHashesHolder: &trieMock.CheckpointHashesHolderStub{}, @@ -1621,9 +1600,20 @@ func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing. } storageManager, _ := trie.NewTrieStorageManager(args) maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(storageManager, marshalizer, hsh, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) - adb, _ := state.NewAccountsDB(tr, hsh, marshalizer, factory.NewAccountCreator(), spm, common.Normal) + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + + adb, _ := state.NewAccountsDB(argsAccountsDB) addr := make([]byte, 32) acc, _ := adb.LoadAccount(addr) @@ -1683,16 +1673,16 @@ func TestAccountsDB_RemoveAccountMarksObsoleteHashesForEviction(t *testing.T) { t.Parallel() maxTrieLevelInMemory := uint(5) - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} - ewl := stateMock.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshalizer) + ewl := stateMock.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshaller) args := trie.NewTrieStorageManagerArgs{ DB: testscommon.NewMemDbMock(), MainStorer: testscommon.CreateMemUnit(), CheckpointsStorer: testscommon.CreateMemUnit(), - Marshalizer: marshalizer, - Hasher: hsh, + Marshalizer: marshaller, + Hasher: hasher, SnapshotDbConfig: config.DBConfig{}, GeneralConfig: config.TrieStorageManagerConfig{SnapshotsGoroutineNum: 1}, CheckpointHashesHolder: &trieMock.CheckpointHashesHolderStub{}, @@ -1700,9 +1690,19 @@ func TestAccountsDB_RemoveAccountMarksObsoleteHashesForEviction(t *testing.T) { IdleProvider: &testscommon.ProcessStatusHandlerStub{}, } storageManager, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(storageManager, marshalizer, hsh, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) - adb, _ := state.NewAccountsDB(tr, hsh, marshalizer, factory.NewAccountCreator(), spm, common.Normal) + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) addr := make([]byte, 32) acc, _ := adb.LoadAccount(addr) @@ -2107,14 +2107,14 @@ func TestAccountsDB_GetCode(t *testing.T) { t.Parallel() maxTrieLevelInMemory := uint(5) - marshalizer := &testscommon.MarshalizerMock{} + marshaller := &testscommon.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} args := trie.NewTrieStorageManagerArgs{ DB: testscommon.NewMemDbMock(), MainStorer: testscommon.CreateMemUnit(), CheckpointsStorer: testscommon.CreateMemUnit(), - Marshalizer: marshalizer, + Marshalizer: marshaller, Hasher: hasher, SnapshotDbConfig: config.DBConfig{}, GeneralConfig: config.TrieStorageManagerConfig{SnapshotsGoroutineNum: 1}, @@ -2123,9 +2123,18 @@ func TestAccountsDB_GetCode(t *testing.T) { IdleProvider: &testscommon.ProcessStatusHandlerStub{}, } storageManager, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(storageManager, marshalizer, hasher, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm := disabled.NewDisabledStoragePruningManager() - adb, _ := state.NewAccountsDB(tr, hasher, marshalizer, factory.NewAccountCreator(), spm, common.Normal) + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) address := make([]byte, 32) acc, err := adb.LoadAccount(address) @@ -2229,11 +2238,20 @@ func TestAccountsDB_Close(t *testing.T) { return &testscommon.StorageManagerStub{} }, } - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} - ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshalizer) + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} + ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshaller) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) - adb, _ := state.NewAccountsDB(tr, hsh, marshalizer, factory.NewAccountCreator(), spm, common.Normal) + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) err := adb.Close() assert.Nil(t, err) @@ -2253,10 +2271,10 @@ func TestAccountsDB_GetAccountFromBytesInvalidAddress(t *testing.T) { func TestAccountsDB_GetAccountFromBytes(t *testing.T) { t.Parallel() - marshalizer := &testscommon.MarshalizerMock{} + marshaller := &testscommon.MarshalizerMock{} adr := make([]byte, 32) accountExpected, _ := state.NewUserAccount(adr) - accountBytes, _ := marshalizer.Marshal(accountExpected) + accountBytes, _ := marshaller.Marshal(accountExpected) _, adb := getDefaultTrieAndAccountsDb() acc, err := adb.GetAccountFromBytes(adr, accountBytes) @@ -2270,8 +2288,8 @@ func TestAccountsDB_GetAccountFromBytesShouldLoadDataTrie(t *testing.T) { acc := generateAccount() acc.SetRootHash([]byte("root hash")) dataTrie := &trieMock.TrieStub{} - marshalizer := &testscommon.MarshalizerMock{} - serializerAcc, _ := marshalizer.Marshal(acc) + marshaller := &testscommon.MarshalizerMock{} + serializerAcc, _ := marshaller.Marshal(acc) trieStub := &trieMock.TrieStub{ GetCalled: func(key []byte) (i []byte, e error) { @@ -2330,14 +2348,14 @@ func TestAccountsDB_NewAccountsDbStartsSnapshotAfterRestart(t *testing.T) { func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { maxTrieLevelInMemory := uint(5) - marshalizer := &testscommon.MarshalizerMock{} + marshaller := &testscommon.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} args := trie.NewTrieStorageManagerArgs{ DB: testscommon.NewMemDbMock(), MainStorer: testscommon.CreateMemUnit(), CheckpointsStorer: testscommon.CreateMemUnit(), - Marshalizer: marshalizer, + Marshalizer: marshaller, Hasher: hasher, SnapshotDbConfig: config.DBConfig{}, GeneralConfig: config.TrieStorageManagerConfig{SnapshotsGoroutineNum: 1}, @@ -2346,9 +2364,18 @@ func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { IdleProvider: &testscommon.ProcessStatusHandlerStub{}, } storageManager, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(storageManager, marshalizer, hasher, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm := disabled.NewDisabledStoragePruningManager() - adb, _ := state.NewAccountsDB(tr, hasher, marshalizer, factory.NewAccountCreator(), spm, common.Normal) + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) address := make([]byte, 32) acc, err := adb.LoadAccount(address) @@ -2370,7 +2397,47 @@ func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - entry, _ := state.GetCodeEntry(codeHash, tr, marshalizer) + entry, _ := state.GetCodeEntry(codeHash, tr, marshaller) assert.Equal(b, code, entry.Code) } } + +func TestAccountsDB_waitForCompletionIfRunningInImportDB(t *testing.T) { + t.Parallel() + + t.Run("not in import db", func(t *testing.T) { + t.Parallel() + + argsAccountsDB := createMockAccountsDBArgs() + argsAccountsDB.ProcessingMode = common.Normal + argsAccountsDB.ProcessStatusHandler = &testscommon.ProcessStatusHandlerStub{ + SetIdleCalled: func() { + assert.Fail(t, "should have not called set idle ") + }, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) + adb.WaitForCompletionIfRunningInImportDB(&trieMock.MockStatistics{}) + }) + t.Run("in import db", func(t *testing.T) { + t.Parallel() + + argsAccountsDB := createMockAccountsDBArgs() + argsAccountsDB.ProcessingMode = common.ImportDb + idleWasSet := false + argsAccountsDB.ProcessStatusHandler = &testscommon.ProcessStatusHandlerStub{ + SetIdleCalled: func() { + idleWasSet = true + }, + } + + stats := &trieMock.MockStatistics{} + waitForSnapshotsToFinishCalled := false + stats.WaitForSnapshotsToFinishCalled = func() { + waitForSnapshotsToFinishCalled = true + } + adb, _ := state.NewAccountsDB(argsAccountsDB) + adb.WaitForCompletionIfRunningInImportDB(stats) + assert.True(t, idleWasSet) + assert.True(t, waitForSnapshotsToFinishCalled) + }) +} diff --git a/state/errors.go b/state/errors.go index 966de871029..d85a738bdb7 100644 --- a/state/errors.go +++ b/state/errors.go @@ -121,3 +121,6 @@ var ErrNilRootHash = errors.New("nil root hash") // ErrNilChainHandler signals that a nil chain handler was provided var ErrNilChainHandler = errors.New("nil chain handler") + +// ErrNilProcessStatusHandler signals that a nil process status handler was provided +var ErrNilProcessStatusHandler = errors.New("nil process status handler") diff --git a/state/export_test.go b/state/export_test.go index 6161fdf3524..91a652573d5 100644 --- a/state/export_test.go +++ b/state/export_test.go @@ -2,9 +2,11 @@ package state import ( "github.com/ElrondNetwork/elrond-go-core/marshal" + "github.com/ElrondNetwork/elrond-go/common" vmcommon "github.com/ElrondNetwork/elrond-vm-common" ) +// NewEmptyBaseAccount - func NewEmptyBaseAccount(address []byte, tracker DataTrieTracker) *baseAccount { return &baseAccount{ address: address, @@ -12,26 +14,37 @@ func NewEmptyBaseAccount(address []byte, tracker DataTrieTracker) *baseAccount { } } +// LoadCode - func (adb *AccountsDB) LoadCode(accountHandler baseAccountHandler) error { return adb.loadCode(accountHandler) } +// LoadDataTrie - func (adb *AccountsDB) LoadDataTrie(accountHandler baseAccountHandler) error { return adb.loadDataTrie(accountHandler) } +// GetAccount - func (adb *AccountsDB) GetAccount(address []byte) (vmcommon.AccountHandler, error) { return adb.getAccount(address) } +// GetObsoleteHashes - func (adb *AccountsDB) GetObsoleteHashes() map[string][][]byte { return adb.obsoleteDataTrieHashes } +// GetObsoleteHashes - +func (adb *AccountsDB) WaitForCompletionIfRunningInImportDB(stats common.SnapshotStatisticsHandler) { + adb.waitForCompletionIfRunningInImportDB(stats) +} + +// GetCode - func GetCode(account baseAccountHandler) []byte { return account.GetCodeHash() } +// GetCodeEntry - func GetCodeEntry(codeHash []byte, trie Updater, marshalizer marshal.Marshalizer) (*CodeEntry, error) { return getCodeEntry(codeHash, trie, marshalizer) } diff --git a/state/journalEntries_test.go b/state/journalEntries_test.go index e99e7cb81dd..20b21eea997 100644 --- a/state/journalEntries_test.go +++ b/state/journalEntries_test.go @@ -219,7 +219,6 @@ func TestJournalEntryDataTrieUpdates_RevertFailsWhenUpdateFails(t *testing.T) { } accnt.SetDataTrie(tr) - //accnt, _ := state.NewAccount(mock.NewAddressMock(), &mock.AccountTrackerStub{}) entry, _ := state.NewJournalEntryDataTrieUpdates(trieUpdates, accnt) acc, err := entry.Revert() diff --git a/state/peerAccount.go b/state/peerAccount.go index 6c6eea4e1a4..18a5ca1a280 100644 --- a/state/peerAccount.go +++ b/state/peerAccount.go @@ -141,7 +141,7 @@ func (pa *peerAccount) SetConsecutiveProposerMisses(consecutiveMisses uint32) { pa.ConsecutiveProposerMisses = consecutiveMisses } -//IncreaseNonce adds the given value to the current nonce +// IncreaseNonce adds the given value to the current nonce func (pa *peerAccount) IncreaseNonce(value uint64) { pa.Nonce = pa.Nonce + value } diff --git a/state/peerAccountsDB.go b/state/peerAccountsDB.go index 1ea39087206..629f02677df 100644 --- a/state/peerAccountsDB.go +++ b/state/peerAccountsDB.go @@ -5,9 +5,6 @@ import ( "fmt" "sync" - "github.com/ElrondNetwork/elrond-go-core/core/check" - "github.com/ElrondNetwork/elrond-go-core/hashing" - "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" ) @@ -17,37 +14,20 @@ type PeerAccountsDB struct { } // NewPeerAccountsDB creates a new account manager -func NewPeerAccountsDB( - trie common.Trie, - hasher hashing.Hasher, - marshalizer marshal.Marshalizer, - accountFactory AccountFactory, - storagePruningManager StoragePruningManager, -) (*PeerAccountsDB, error) { - if check.IfNil(trie) { - return nil, ErrNilTrie - } - if check.IfNil(hasher) { - return nil, ErrNilHasher - } - if check.IfNil(marshalizer) { - return nil, ErrNilMarshalizer - } - if check.IfNil(accountFactory) { - return nil, ErrNilAccountFactory - } - if check.IfNil(storagePruningManager) { - return nil, ErrNilStoragePruningManager +func NewPeerAccountsDB(args ArgsAccountsDB) (*PeerAccountsDB, error) { + err := checkArgsAccountsDB(args) + if err != nil { + return nil, err } - trieStorageManager := trie.GetStorageManager() + trieStorageManager := args.Trie.GetStorageManager() numCheckpoints := getNumCheckpoints(trieStorageManager) adb := &PeerAccountsDB{ &AccountsDB{ - mainTrie: trie, - hasher: hasher, - marshalizer: marshalizer, - accountFactory: accountFactory, + mainTrie: args.Trie, + hasher: args.Hasher, + marshaller: args.Marshaller, + accountFactory: args.AccountFactory, entries: make([]JournalEntry, 0), dataTries: NewDataTriesHolder(), mutOp: sync.RWMutex{}, @@ -55,8 +35,10 @@ func NewPeerAccountsDB( loadCodeMeasurements: &loadingMeasurements{ identifier: "load code", }, - storagePruningManager: storagePruningManager, + storagePruningManager: args.StoragePruningManager, + processingMode: args.ProcessingMode, lastSnapshot: &snapshotInfo{}, + processStatusHandler: args.ProcessStatusHandler, }, } @@ -127,9 +109,7 @@ func (adb *PeerAccountsDB) SnapshotState(rootHash []byte) { handleLoggingWhenError("error while putting active DB value into main storer", err) }() - if adb.processingMode == common.ImportDb { - stats.WaitForSnapshotsToFinish() - } + adb.waitForCompletionIfRunningInImportDB(stats) adb.increaseNumCheckpoints() } @@ -148,9 +128,7 @@ func (adb *PeerAccountsDB) SetStateCheckpoint(rootHash []byte) { go printStats(stats, "snapshotState peer trie", rootHash) - if adb.processingMode == common.ImportDb { - stats.WaitForSnapshotsToFinish() - } + adb.waitForCompletionIfRunningInImportDB(stats) adb.increaseNumCheckpoints() } diff --git a/state/peerAccountsDB_test.go b/state/peerAccountsDB_test.go index 9231070c872..7158d95635f 100644 --- a/state/peerAccountsDB_test.go +++ b/state/peerAccountsDB_test.go @@ -12,127 +12,110 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/check" "github.com/ElrondNetwork/elrond-go/common" "github.com/ElrondNetwork/elrond-go/state" - "github.com/ElrondNetwork/elrond-go/state/storagePruningManager/disabled" "github.com/ElrondNetwork/elrond-go/testscommon" - "github.com/ElrondNetwork/elrond-go/testscommon/hashingMocks" - stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" trieMock "github.com/ElrondNetwork/elrond-go/testscommon/trie" "github.com/stretchr/testify/assert" ) -func TestNewPeerAccountsDB_WithNilTrieShouldErr(t *testing.T) { +func TestNewPeerAccountsDB(t *testing.T) { t.Parallel() - adb, err := state.NewPeerAccountsDB( - nil, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + t.Run("nil trie should error", func(t *testing.T) { + t.Parallel() - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilTrie, err) -} + args := createMockAccountsDBArgs() + args.Trie = nil -func TestNewPeerAccountsDB_WithNilHasherShouldErr(t *testing.T) { - t.Parallel() + adb, err := state.NewPeerAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilTrie, err) + }) + t.Run("nil hasher should error", func(t *testing.T) { + t.Parallel() - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{}, - nil, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + args := createMockAccountsDBArgs() + args.Hasher = nil - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilHasher, err) -} + adb, err := state.NewPeerAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilHasher, err) + }) + t.Run("nil hasher should error", func(t *testing.T) { + t.Parallel() -func TestNewPeerAccountsDB_WithNilMarshalizerShouldErr(t *testing.T) { - t.Parallel() + args := createMockAccountsDBArgs() + args.Hasher = nil - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{}, - &hashingMocks.HasherMock{}, - nil, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + adb, err := state.NewPeerAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilHasher, err) + }) + t.Run("nil marshaller should error", func(t *testing.T) { + t.Parallel() - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilMarshalizer, err) -} + args := createMockAccountsDBArgs() + args.Marshaller = nil -func TestNewPeerAccountsDB_WithNilAddressFactoryShouldErr(t *testing.T) { - t.Parallel() + adb, err := state.NewPeerAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilMarshalizer, err) + }) + t.Run("nil account factory should error", func(t *testing.T) { + t.Parallel() - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{}, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - nil, - disabled.NewDisabledStoragePruningManager(), - ) + args := createMockAccountsDBArgs() + args.AccountFactory = nil - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilAccountFactory, err) -} + adb, err := state.NewPeerAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilAccountFactory, err) + }) + t.Run("nil storage pruning manager should error", func(t *testing.T) { + t.Parallel() -func TestNewPeerAccountsDB_WithNilStoragePruningManagerShouldErr(t *testing.T) { - t.Parallel() + args := createMockAccountsDBArgs() + args.StoragePruningManager = nil - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{}, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - nil, - ) + adb, err := state.NewPeerAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilStoragePruningManager, err) + }) + t.Run("nil process status handler should error", func(t *testing.T) { + t.Parallel() - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilStoragePruningManager, err) -} + args := createMockAccountsDBArgs() + args.ProcessStatusHandler = nil -func TestNewPeerAccountsDB_OkValsShouldWork(t *testing.T) { - t.Parallel() + adb, err := state.NewPeerAccountsDB(args) + assert.True(t, check.IfNil(adb)) + assert.Equal(t, state.ErrNilProcessStatusHandler, err) + }) + t.Run("should work", func(t *testing.T) { + t.Parallel() - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{} - }, - }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + args := createMockAccountsDBArgs() - assert.Nil(t, err) - assert.False(t, check.IfNil(adb)) + adb, err := state.NewPeerAccountsDB(args) + assert.False(t, check.IfNil(adb)) + assert.Nil(t, err) + }) } func TestNewPeerAccountsDB_SnapshotState(t *testing.T) { t.Parallel() snapshotCalled := false - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{ - TakeSnapshotCalled: func(_ []byte, _ []byte, _ chan core.KeyValueHolder, _ common.SnapshotStatisticsHandler, _ uint32) { - snapshotCalled = true - }, - } - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{ + TakeSnapshotCalled: func(_ []byte, _ []byte, _ chan core.KeyValueHolder, _ common.SnapshotStatisticsHandler, _ uint32) { + snapshotCalled = true + }, + } }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + } + adb, err := state.NewPeerAccountsDB(args) assert.Nil(t, err) assert.False(t, check.IfNil(adb)) @@ -145,24 +128,20 @@ func TestNewPeerAccountsDB_SnapshotStateGetLatestStorageEpochErrDoesNotSnapshot( t.Parallel() snapshotCalled := false - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{ - GetLatestStorageEpochCalled: func() (uint32, error) { - return 0, fmt.Errorf("new error") - }, - TakeSnapshotCalled: func(_ []byte, _ []byte, _ chan core.KeyValueHolder, _ common.SnapshotStatisticsHandler, _ uint32) { - snapshotCalled = true - }, - } - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{ + GetLatestStorageEpochCalled: func() (uint32, error) { + return 0, fmt.Errorf("new error") + }, + TakeSnapshotCalled: func(_ []byte, _ []byte, _ chan core.KeyValueHolder, _ common.SnapshotStatisticsHandler, _ uint32) { + snapshotCalled = true + }, + } }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + } + adb, err := state.NewPeerAccountsDB(args) assert.Nil(t, err) assert.False(t, check.IfNil(adb)) @@ -174,21 +153,17 @@ func TestNewPeerAccountsDB_SetStateCheckpoint(t *testing.T) { t.Parallel() checkpointCalled := false - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{ - SetCheckpointCalled: func(_ []byte, _ []byte, _ chan core.KeyValueHolder, _ common.SnapshotStatisticsHandler) { - checkpointCalled = true - }, - } - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{ + SetCheckpointCalled: func(_ []byte, _ []byte, _ chan core.KeyValueHolder, _ common.SnapshotStatisticsHandler) { + checkpointCalled = true + }, + } }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + } + adb, err := state.NewPeerAccountsDB(args) assert.Nil(t, err) assert.False(t, check.IfNil(adb)) @@ -201,21 +176,17 @@ func TestNewPeerAccountsDB_RecreateAllTries(t *testing.T) { t.Parallel() recreateCalled := false - adb, err := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{} - }, - RecreateCalled: func(_ []byte) (common.Trie, error) { - recreateCalled = true - return nil, nil - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{} + }, + RecreateCalled: func(_ []byte) (common.Trie, error) { + recreateCalled = true + return nil, nil }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + } + adb, err := state.NewPeerAccountsDB(args) assert.Nil(t, err) assert.False(t, check.IfNil(adb)) @@ -256,13 +227,9 @@ func TestPeerAccountsDB_NewAccountsDbStartsSnapshotAfterRestart(t *testing.T) { }, } - adb, err := state.NewPeerAccountsDB( - trieStub, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + args := createMockAccountsDBArgs() + args.Trie = trieStub + adb, err := state.NewPeerAccountsDB(args) assert.Nil(t, err) assert.NotNil(t, adb) @@ -286,25 +253,21 @@ func TestPeerAccountsDB_MarkSnapshotDone(t *testing.T) { }() expectedErr := errors.New("expected error") - adb, _ := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{ - PutInEpochCalled: func(bytes []byte, bytes2 []byte, u uint32) error { - assert.Fail(t, "should have not called put in epoch") - return nil - }, - GetLatestStorageEpochCalled: func() (uint32, error) { - return 0, expectedErr - }, - } - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{ + PutInEpochCalled: func(bytes []byte, bytes2 []byte, u uint32) error { + assert.Fail(t, "should have not called put in epoch") + return nil + }, + GetLatestStorageEpochCalled: func() (uint32, error) { + return 0, expectedErr + }, + } }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + } + adb, _ := state.NewPeerAccountsDB(args) adb.MarkSnapshotDone() }) @@ -320,25 +283,21 @@ func TestPeerAccountsDB_MarkSnapshotDone(t *testing.T) { expectedErr := errors.New("expected error") putWasCalled := false - adb, _ := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{ - PutInEpochCalled: func(key []byte, value []byte, epoch uint32) error { - assert.Equal(t, common.ActiveDBKey, string(key)) - assert.Equal(t, common.ActiveDBVal, string(value)) - putWasCalled = true - - return expectedErr - }, - } - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{ + PutInEpochCalled: func(key []byte, value []byte, epoch uint32) error { + assert.Equal(t, common.ActiveDBKey, string(key)) + assert.Equal(t, common.ActiveDBVal, string(value)) + putWasCalled = true + + return expectedErr + }, + } }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + } + adb, _ := state.NewPeerAccountsDB(args) adb.MarkSnapshotDone() assert.True(t, putWasCalled) @@ -347,25 +306,21 @@ func TestPeerAccountsDB_MarkSnapshotDone(t *testing.T) { t.Parallel() putWasCalled := false - adb, _ := state.NewPeerAccountsDB( - &trieMock.TrieStub{ - GetStorageManagerCalled: func() common.StorageManager { - return &testscommon.StorageManagerStub{ - PutInEpochCalled: func(key []byte, value []byte, epoch uint32) error { - assert.Equal(t, common.ActiveDBKey, string(key)) - assert.Equal(t, common.ActiveDBVal, string(value)) - putWasCalled = true - - return nil - }, - } - }, + args := createMockAccountsDBArgs() + args.Trie = &trieMock.TrieStub{ + GetStorageManagerCalled: func() common.StorageManager { + return &testscommon.StorageManagerStub{ + PutInEpochCalled: func(key []byte, value []byte, epoch uint32) error { + assert.Equal(t, common.ActiveDBKey, string(key)) + assert.Equal(t, common.ActiveDBVal, string(value)) + putWasCalled = true + + return nil + }, + } }, - &hashingMocks.HasherMock{}, - &testscommon.MarshalizerMock{}, - &stateMock.AccountsFactoryStub{}, - disabled.NewDisabledStoragePruningManager(), - ) + } + adb, _ := state.NewPeerAccountsDB(args) adb.MarkSnapshotDone() assert.True(t, putWasCalled) diff --git a/state/storagePruningManager/storagePruningManager_test.go b/state/storagePruningManager/storagePruningManager_test.go index a789477c747..aae399c0b0c 100644 --- a/state/storagePruningManager/storagePruningManager_test.go +++ b/state/storagePruningManager/storagePruningManager_test.go @@ -23,14 +23,14 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. MaxSnapshots: 2, SnapshotsGoroutineNum: 1, } - marshalizer := &testscommon.MarshalizerMock{} - hsh := &hashingMocks.HasherMock{} + marshaller := &testscommon.MarshalizerMock{} + hasher := &hashingMocks.HasherMock{} args := trie.NewTrieStorageManagerArgs{ DB: testscommon.NewMemDbMock(), MainStorer: testscommon.CreateMemUnit(), CheckpointsStorer: testscommon.CreateMemUnit(), - Marshalizer: marshalizer, - Hasher: hsh, + Marshalizer: marshaller, + Hasher: hasher, SnapshotDbConfig: config.DBConfig{ Type: "MemoryDB", }, @@ -40,10 +40,20 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. IdleProvider: &testscommon.ProcessStatusHandlerStub{}, } trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshalizer, hsh, 5) - ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshalizer) + tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, 5) + ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshaller) spm, _ := NewStoragePruningManager(ewl, generalCfg.PruningBufferLen) - adb, _ := state.NewAccountsDB(tr, hsh, marshalizer, factory.NewAccountCreator(), spm, common.Normal) + + argsAccountsDB := state.ArgsAccountsDB{ + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: factory.NewAccountCreator(), + StoragePruningManager: spm, + ProcessingMode: common.Normal, + ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, + } + adb, _ := state.NewAccountsDB(argsAccountsDB) return tr, adb, spm } diff --git a/state/trackableDataTrie.go b/state/trackableDataTrie.go index cef876122a3..233ca4f487f 100644 --- a/state/trackableDataTrie.go +++ b/state/trackableDataTrie.go @@ -38,13 +38,13 @@ func (tdaw *TrackableDataTrie) DirtyData() map[string][]byte { func (tdaw *TrackableDataTrie) RetrieveValue(key []byte) ([]byte, error) { tailLength := len(key) + len(tdaw.identifier) - //search in dirty data cache + // search in dirty data cache if value, found := tdaw.dirtyData[string(key)]; found { log.Trace("retrieve value from dirty data", "key", key, "value", value) return trimValue(value, tailLength) } - //ok, not in cache, retrieve from trie + // ok, not in cache, retrieve from trie if tdaw.tr == nil { return nil, ErrNilTrie } diff --git a/state/trackableDataTrie_test.go b/state/trackableDataTrie_test.go index eb4b7f7c77d..2ded76c2b39 100644 --- a/state/trackableDataTrie_test.go +++ b/state/trackableDataTrie_test.go @@ -158,7 +158,7 @@ func TestTrackableDataTrie_SaveKeyValueShouldSaveOnlyInDirty(t *testing.T) { _ = mdaw.SaveKeyValue(keyExpected, value) - //test in dirty + // test in dirty assert.Equal(t, expectedVal, mdaw.DirtyData()[string(keyExpected)]) } @@ -173,11 +173,11 @@ func TestTrackableDataTrie_ClearDataCachesValidDataShouldWork(t *testing.T) { assert.Equal(t, 0, len(mdaw.DirtyData())) - //add something + // add something _ = mdaw.SaveKeyValue([]byte("ABC"), []byte("123")) assert.Equal(t, 1, len(mdaw.DirtyData())) - //clear + // clear mdaw.ClearDataCaches() assert.Equal(t, 0, len(mdaw.DirtyData())) } diff --git a/state/userAccount.go b/state/userAccount.go index b30754767b4..6c0f3bf7d3a 100644 --- a/state/userAccount.go +++ b/state/userAccount.go @@ -120,7 +120,7 @@ func (a *userAccount) SetOwnerAddress(address []byte) { a.OwnerAddress = address } -//IncreaseNonce adds the given value to the current nonce +// IncreaseNonce adds the given value to the current nonce func (a *userAccount) IncreaseNonce(value uint64) { a.Nonce = a.Nonce + value } diff --git a/testscommon/trie/statisticsMock.go b/testscommon/trie/statisticsMock.go index 9af3f501f00..334390f3998 100644 --- a/testscommon/trie/statisticsMock.go +++ b/testscommon/trie/statisticsMock.go @@ -2,9 +2,10 @@ package trie // MockStatistics - type MockStatistics struct { - size uint64 - numDataTries uint64 - numNodes uint64 + size uint64 + numDataTries uint64 + numNodes uint64 + WaitForSnapshotsToFinishCalled func() } // AddSize - @@ -28,4 +29,7 @@ func (m *MockStatistics) NewDataTrie() { // WaitForSnapshotsToFinish - func (m *MockStatistics) WaitForSnapshotsToFinish() { + if m.WaitForSnapshotsToFinishCalled != nil { + m.WaitForSnapshotsToFinishCalled() + } } diff --git a/update/genesis/import.go b/update/genesis/import.go index 49db625f1ea..482d75264a1 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -14,6 +14,7 @@ import ( "github.com/ElrondNetwork/elrond-go-core/hashing" "github.com/ElrondNetwork/elrond-go-core/marshal" "github.com/ElrondNetwork/elrond-go/common" + commonDisabled "github.com/ElrondNetwork/elrond-go/common/disabled" "github.com/ElrondNetwork/elrond-go/config" "github.com/ElrondNetwork/elrond-go/state" "github.com/ElrondNetwork/elrond-go/state/factory" @@ -393,14 +394,16 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32) (state.Accoun if accType == ValidatorAccount { if check.IfNil(si.validatorDB) { - accountsDB, errCreate := state.NewAccountsDB( - currentTrie, - si.hasher, - si.marshalizer, - accountFactory, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + argsAccountDB := state.ArgsAccountsDB{ + Trie: currentTrie, + Hasher: si.hasher, + Marshaller: si.marshalizer, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + ProcessingMode: common.Normal, + ProcessStatusHandler: &commonDisabled.ProcessStatusHandler{}, + } + accountsDB, errCreate := state.NewAccountsDB(argsAccountDB) if errCreate != nil { return nil, nil, errCreate } @@ -414,14 +417,16 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32) (state.Accoun return accountsDB, currentTrie, nil } - accountsDB, err = state.NewAccountsDB( - currentTrie, - si.hasher, - si.marshalizer, - accountFactory, - disabled.NewDisabledStoragePruningManager(), - common.Normal, - ) + argsAccountDB := state.ArgsAccountsDB{ + Trie: currentTrie, + Hasher: si.hasher, + Marshaller: si.marshalizer, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + ProcessingMode: common.Normal, + ProcessStatusHandler: &commonDisabled.ProcessStatusHandler{}, + } + accountsDB, err = state.NewAccountsDB(argsAccountDB) si.accountDBsMap[shardID] = accountsDB return accountsDB, currentTrie, err } From 99cd1424a7132ed57006c5c9735d4ca957c60cd4 Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Thu, 14 Apr 2022 13:03:08 +0300 Subject: [PATCH 02/19] - fixes after self review --- state/accountsDB_test.go | 10 ---------- state/export_test.go | 2 +- state/peerAccountsDB_test.go | 10 ---------- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index c5e0f0eaafd..0f80d12fe7a 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -151,16 +151,6 @@ func TestNewAccountsDB(t *testing.T) { assert.True(t, check.IfNil(adb)) assert.Equal(t, state.ErrNilHasher, err) }) - t.Run("nil hasher should error", func(t *testing.T) { - t.Parallel() - - args := createMockAccountsDBArgs() - args.Hasher = nil - - adb, err := state.NewAccountsDB(args) - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilHasher, err) - }) t.Run("nil marshaller should error", func(t *testing.T) { t.Parallel() diff --git a/state/export_test.go b/state/export_test.go index 91a652573d5..088405fc533 100644 --- a/state/export_test.go +++ b/state/export_test.go @@ -34,7 +34,7 @@ func (adb *AccountsDB) GetObsoleteHashes() map[string][][]byte { return adb.obsoleteDataTrieHashes } -// GetObsoleteHashes - +// WaitForCompletionIfRunningInImportDB - func (adb *AccountsDB) WaitForCompletionIfRunningInImportDB(stats common.SnapshotStatisticsHandler) { adb.waitForCompletionIfRunningInImportDB(stats) } diff --git a/state/peerAccountsDB_test.go b/state/peerAccountsDB_test.go index 7158d95635f..50597bdbe6e 100644 --- a/state/peerAccountsDB_test.go +++ b/state/peerAccountsDB_test.go @@ -40,16 +40,6 @@ func TestNewPeerAccountsDB(t *testing.T) { assert.True(t, check.IfNil(adb)) assert.Equal(t, state.ErrNilHasher, err) }) - t.Run("nil hasher should error", func(t *testing.T) { - t.Parallel() - - args := createMockAccountsDBArgs() - args.Hasher = nil - - adb, err := state.NewPeerAccountsDB(args) - assert.True(t, check.IfNil(adb)) - assert.Equal(t, state.ErrNilHasher, err) - }) t.Run("nil marshaller should error", func(t *testing.T) { t.Parallel() From 603ea557caf0cb7fc8cc6398a8d3028f54a0fccf Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Thu, 14 Apr 2022 14:45:07 +0300 Subject: [PATCH 03/19] - fixes after reviews --- common/disabled/processStatusHandler.go | 17 ++-- common/disabled/processStatusHandler_test.go | 26 ++++++ .../baseResolversContainerFactory.go | 2 +- genesis/process/memoryComponents.go | 2 +- state/accountsDB_test.go | 85 +++++++++---------- update/genesis/import.go | 4 +- 6 files changed, 79 insertions(+), 57 deletions(-) create mode 100644 common/disabled/processStatusHandler_test.go diff --git a/common/disabled/processStatusHandler.go b/common/disabled/processStatusHandler.go index 33747f9bc76..036075099af 100644 --- a/common/disabled/processStatusHandler.go +++ b/common/disabled/processStatusHandler.go @@ -1,22 +1,27 @@ package disabled -// ProcessStatusHandler is the disabled implementation for the status handler that keeps track what the node is doing: +// processStatusHandler is the disabled implementation for the status handler that keeps track what the node is doing: // processing blocks or idle -type ProcessStatusHandler struct { +type processStatusHandler struct { +} + +// NewProcessStatusHandler creates a new instance of type processStatusHandler +func NewProcessStatusHandler() *processStatusHandler { + return &processStatusHandler{} } // SetBusy does nothing -func (psh *ProcessStatusHandler) SetBusy(_ string) {} +func (psh *processStatusHandler) SetBusy(_ string) {} // SetIdle does nothing -func (psh *ProcessStatusHandler) SetIdle() {} +func (psh *processStatusHandler) SetIdle() {} // IsIdle returns true -func (psh *ProcessStatusHandler) IsIdle() bool { +func (psh *processStatusHandler) IsIdle() bool { return true } // IsInterfaceNil returns true if there is no value under the interface -func (psh *ProcessStatusHandler) IsInterfaceNil() bool { +func (psh *processStatusHandler) IsInterfaceNil() bool { return psh == nil } diff --git a/common/disabled/processStatusHandler_test.go b/common/disabled/processStatusHandler_test.go new file mode 100644 index 00000000000..b63f82a3bbd --- /dev/null +++ b/common/disabled/processStatusHandler_test.go @@ -0,0 +1,26 @@ +package disabled + +import ( + "fmt" + "testing" + + "github.com/ElrondNetwork/elrond-go-core/core/check" + "github.com/stretchr/testify/assert" +) + +func TestProcessStatusHandler_MethodsShouldNotPanic(t *testing.T) { + t.Parallel() + + defer func() { + r := recover() + if r != nil { + assert.Fail(t, fmt.Sprintf("should have not panicked: %v", r)) + } + }() + + psh := NewProcessStatusHandler() + assert.False(t, check.IfNil(psh)) + psh.SetBusy("") + psh.SetIdle() + assert.True(t, psh.IsIdle()) +} diff --git a/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go b/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go index f14d4cfdd79..eaf7e328c7c 100644 --- a/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go +++ b/dataRetriever/factory/storageResolversContainer/baseResolversContainerFactory.go @@ -233,7 +233,7 @@ func (brcf *baseResolversContainerFactory) newImportDBTrieStorage( MaxTrieLevelInMem: brcf.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory, DisableOldTrieStorageEpoch: brcf.disableOldTrieStorageEpoch, EpochStartNotifier: brcf.epochNotifier, - IdleProvider: &disabled.ProcessStatusHandler{}, + IdleProvider: disabled.NewProcessStatusHandler(), } return trieFactoryInstance.Create(args) } diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index 12aaf62ff70..e0f4ef63155 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -30,7 +30,7 @@ func createAccountAdapter( AccountFactory: accountFactory, StoragePruningManager: disabled.NewDisabledStoragePruningManager(), ProcessingMode: common.Normal, - ProcessStatusHandler: &commonDisabled.ProcessStatusHandler{}, + ProcessStatusHandler: commonDisabled.NewProcessStatusHandler(), } adb, err := state.NewAccountsDB(args) diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 0f80d12fe7a..2a236039a17 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -60,8 +60,8 @@ func generateAccountDBFromTrie(trie common.Trie) *state.AccountsDB { args := createMockAccountsDBArgs() args.Trie = trie - accnt, _ := state.NewAccountsDB(args) - return accnt + accounts, _ := state.NewAccountsDB(args) + return accounts } func generateAccount() *stateMock.AccountWrapMock { @@ -1593,15 +1593,12 @@ func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing. tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) - argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: factory.NewAccountCreator(), - StoragePruningManager: spm, - ProcessingMode: common.Normal, - ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, - } + argsAccountsDB := createMockAccountsDBArgs() + argsAccountsDB.Trie = tr + argsAccountsDB.Hasher = hasher + argsAccountsDB.Marshaller = marshaller + argsAccountsDB.AccountFactory = factory.NewAccountCreator() + argsAccountsDB.StoragePruningManager = spm adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -1683,15 +1680,13 @@ func TestAccountsDB_RemoveAccountMarksObsoleteHashesForEviction(t *testing.T) { tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) - argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: factory.NewAccountCreator(), - StoragePruningManager: spm, - ProcessingMode: common.Normal, - ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, - } + argsAccountsDB := createMockAccountsDBArgs() + argsAccountsDB.Trie = tr + argsAccountsDB.Hasher = hasher + argsAccountsDB.Marshaller = marshaller + argsAccountsDB.AccountFactory = factory.NewAccountCreator() + argsAccountsDB.StoragePruningManager = spm + adb, _ := state.NewAccountsDB(argsAccountsDB) addr := make([]byte, 32) @@ -2115,15 +2110,13 @@ func TestAccountsDB_GetCode(t *testing.T) { storageManager, _ := trie.NewTrieStorageManager(args) tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm := disabled.NewDisabledStoragePruningManager() - argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: factory.NewAccountCreator(), - StoragePruningManager: spm, - ProcessingMode: common.Normal, - ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, - } + argsAccountsDB := createMockAccountsDBArgs() + argsAccountsDB.Trie = tr + argsAccountsDB.Hasher = hasher + argsAccountsDB.Marshaller = marshaller + argsAccountsDB.AccountFactory = factory.NewAccountCreator() + argsAccountsDB.StoragePruningManager = spm + adb, _ := state.NewAccountsDB(argsAccountsDB) address := make([]byte, 32) @@ -2232,15 +2225,14 @@ func TestAccountsDB_Close(t *testing.T) { hasher := &hashingMocks.HasherMock{} ewl, _ := evictionWaitingList.NewEvictionWaitingList(100, testscommon.NewMemDbMock(), marshaller) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) - argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: factory.NewAccountCreator(), - StoragePruningManager: spm, - ProcessingMode: common.Normal, - ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, - } + + argsAccountsDB := createMockAccountsDBArgs() + argsAccountsDB.Trie = tr + argsAccountsDB.Hasher = hasher + argsAccountsDB.Marshaller = marshaller + argsAccountsDB.AccountFactory = factory.NewAccountCreator() + argsAccountsDB.StoragePruningManager = spm + adb, _ := state.NewAccountsDB(argsAccountsDB) err := adb.Close() @@ -2356,15 +2348,14 @@ func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { storageManager, _ := trie.NewTrieStorageManager(args) tr, _ := trie.NewTrie(storageManager, marshaller, hasher, maxTrieLevelInMemory) spm := disabled.NewDisabledStoragePruningManager() - argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: factory.NewAccountCreator(), - StoragePruningManager: spm, - ProcessingMode: common.Normal, - ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, - } + + argsAccountsDB := createMockAccountsDBArgs() + argsAccountsDB.Trie = tr + argsAccountsDB.Hasher = hasher + argsAccountsDB.Marshaller = marshaller + argsAccountsDB.AccountFactory = factory.NewAccountCreator() + argsAccountsDB.StoragePruningManager = spm + adb, _ := state.NewAccountsDB(argsAccountsDB) address := make([]byte, 32) diff --git a/update/genesis/import.go b/update/genesis/import.go index 482d75264a1..18a2147b744 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -401,7 +401,7 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32) (state.Accoun AccountFactory: accountFactory, StoragePruningManager: disabled.NewDisabledStoragePruningManager(), ProcessingMode: common.Normal, - ProcessStatusHandler: &commonDisabled.ProcessStatusHandler{}, + ProcessStatusHandler: commonDisabled.NewProcessStatusHandler(), } accountsDB, errCreate := state.NewAccountsDB(argsAccountDB) if errCreate != nil { @@ -424,7 +424,7 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32) (state.Accoun AccountFactory: accountFactory, StoragePruningManager: disabled.NewDisabledStoragePruningManager(), ProcessingMode: common.Normal, - ProcessStatusHandler: &commonDisabled.ProcessStatusHandler{}, + ProcessStatusHandler: commonDisabled.NewProcessStatusHandler(), } accountsDB, err = state.NewAccountsDB(argsAccountDB) si.accountDBsMap[shardID] = accountsDB From b5389d4c471dbd1dbc66ad0be6f4141993ed6c41 Mon Sep 17 00:00:00 2001 From: AdoAdoAdo Date: Mon, 18 Apr 2022 13:49:42 +0300 Subject: [PATCH 04/19] process: add a mapping between originaltx and usertx for relayed txs to help with reverting fees --- genesis/process/disabled/feeHandler.go | 4 ++ process/block/postprocess/feeHandler.go | 76 +++++++++++++++++------- process/interface.go | 1 + process/mock/feeAccumulatorStub.go | 18 ++++-- process/transaction/export_test.go | 3 +- process/transaction/shardProcess.go | 26 +++++--- process/transaction/shardProcess_test.go | 5 +- testscommon/unsignedTxHandlerStub.go | 26 +++++--- 8 files changed, 114 insertions(+), 45 deletions(-) diff --git a/genesis/process/disabled/feeHandler.go b/genesis/process/disabled/feeHandler.go index ef87e86139e..20e00b247ab 100644 --- a/genesis/process/disabled/feeHandler.go +++ b/genesis/process/disabled/feeHandler.go @@ -105,6 +105,10 @@ func (fh *FeeHandler) GetAccumulatedFees() *big.Int { func (fh *FeeHandler) ProcessTransactionFee(_ *big.Int, _ *big.Int, _ []byte) { } +// ProcessTransactionFeeRelayedUserTx does nothing +func (fh *FeeHandler) ProcessTransactionFeeRelayedUserTx(_ *big.Int, _ *big.Int, _ []byte, _ []byte){ +} + // RevertFees does nothing func (fh *FeeHandler) RevertFees(_ [][]byte) { } diff --git a/process/block/postprocess/feeHandler.go b/process/block/postprocess/feeHandler.go index 93753b47634..6da0ad76157 100644 --- a/process/block/postprocess/feeHandler.go +++ b/process/block/postprocess/feeHandler.go @@ -18,10 +18,11 @@ type feeData struct { var zero = big.NewInt(0) type feeHandler struct { - mut sync.RWMutex - mapHashFee map[string]*feeData - accumulatedFees *big.Int - developerFees *big.Int + mut sync.RWMutex + mapDependentHashes map[string][]byte + mapHashFee map[string]*feeData + accumulatedFees *big.Int + developerFees *big.Int } // NewFeeAccumulator constructor for the fee accumulator @@ -30,6 +31,7 @@ func NewFeeAccumulator() (*feeHandler, error) { f.accumulatedFees = big.NewInt(0) f.developerFees = big.NewInt(0) f.mapHashFee = make(map[string]*feeData) + f.mapDependentHashes = make(map[string][]byte) return f, nil } @@ -37,6 +39,7 @@ func NewFeeAccumulator() (*feeHandler, error) { func (f *feeHandler) CreateBlockStarted(gasAndFees scheduled.GasAndFees) { f.mut.Lock() f.mapHashFee = make(map[string]*feeData) + f.mapDependentHashes = make(map[string][]byte) f.accumulatedFees = big.NewInt(0) if gasAndFees.AccumulatedFees != nil { f.accumulatedFees = big.NewInt(0).Set(gasAndFees.AccumulatedFees) @@ -69,6 +72,53 @@ func (f *feeHandler) GetDeveloperFees() *big.Int { // ProcessTransactionFee adds the tx cost to the accumulated amount func (f *feeHandler) ProcessTransactionFee(cost *big.Int, devFee *big.Int, txHash []byte) { + f.mut.Lock() + f.processTransactionFee(cost, devFee, txHash) + f.mut.Unlock() +} + +// ProcessTransactionFeeRelayedUserTx processes the transaction fee for the execution of a relayed user transaction +func (f *feeHandler) ProcessTransactionFeeRelayedUserTx(cost *big.Int, devFee *big.Int, userTxHash []byte, originalTxHash []byte) { + f.mut.Lock() + f.linkRelayedUserTxToOriginalTx(userTxHash, originalTxHash) + f.processTransactionFee(cost, devFee, userTxHash) + f.mut.Unlock() +} + +func (f *feeHandler) linkRelayedUserTxToOriginalTx(userTxHash []byte, originalTxHash []byte) { + f.mapDependentHashes[string(originalTxHash)] = userTxHash +} + +func (f *feeHandler) revertFeesForDependentTxHash(txHash []byte) { + linkedTxHash, ok := f.mapDependentHashes[string(txHash)] + if !ok { + return + } + f.revertFee(linkedTxHash) +} + +// RevertFees reverts the accumulated fees for txHashes +func (f *feeHandler) RevertFees(txHashes [][]byte) { + f.mut.Lock() + defer f.mut.Unlock() + + for _, txHash := range txHashes { + f.revertFeesForDependentTxHash(txHash) + f.revertFee(txHash) + } +} + +func (f *feeHandler) revertFee(txHash []byte) { + fee, ok := f.mapHashFee[string(txHash)] + if !ok { + return + } + f.developerFees.Sub(f.developerFees, fee.devFee) + f.accumulatedFees.Sub(f.accumulatedFees, fee.cost) + delete(f.mapHashFee, string(txHash)) +} + +func (f *feeHandler) processTransactionFee(cost *big.Int, devFee *big.Int, txHash []byte) { if cost == nil { log.Error("nil cost in ProcessTransactionFee", "error", process.ErrNilValue.Error()) return @@ -78,7 +128,6 @@ func (f *feeHandler) ProcessTransactionFee(cost *big.Int, devFee *big.Int, txHas return } - f.mut.Lock() fee, ok := f.mapHashFee[string(txHash)] if !ok { fee = &feeData{ @@ -93,23 +142,6 @@ func (f *feeHandler) ProcessTransactionFee(cost *big.Int, devFee *big.Int, txHas f.mapHashFee[string(txHash)] = fee f.accumulatedFees.Add(f.accumulatedFees, cost) f.developerFees.Add(f.developerFees, devFee) - f.mut.Unlock() -} - -// RevertFees reverts the accumulated fees for txHashes -func (f *feeHandler) RevertFees(txHashes [][]byte) { - f.mut.Lock() - defer f.mut.Unlock() - - for _, txHash := range txHashes { - fee, ok := f.mapHashFee[string(txHash)] - if !ok { - continue - } - f.developerFees.Sub(f.developerFees, fee.devFee) - f.accumulatedFees.Sub(f.accumulatedFees, fee.cost) - delete(f.mapHashFee, string(txHash)) - } } // IsInterfaceNil returns true if there is no value under the interface diff --git a/process/interface.go b/process/interface.go index 1ed2de33e56..fe10ab470c4 100644 --- a/process/interface.go +++ b/process/interface.go @@ -195,6 +195,7 @@ type TransactionFeeHandler interface { GetAccumulatedFees() *big.Int GetDeveloperFees() *big.Int ProcessTransactionFee(cost *big.Int, devFee *big.Int, txHash []byte) + ProcessTransactionFeeRelayedUserTx(cost *big.Int, devFee *big.Int, userTxHash []byte, originalTxHash []byte) RevertFees(txHashes [][]byte) IsInterfaceNil() bool } diff --git a/process/mock/feeAccumulatorStub.go b/process/mock/feeAccumulatorStub.go index c390e5662c4..bb8afaa0056 100644 --- a/process/mock/feeAccumulatorStub.go +++ b/process/mock/feeAccumulatorStub.go @@ -8,11 +8,12 @@ import ( // FeeAccumulatorStub is a stub which implements TransactionFeeHandler interface type FeeAccumulatorStub struct { - CreateBlockStartedCalled func(gasAndFees scheduled.GasAndFees) - GetAccumulatedFeesCalled func() *big.Int - GetDeveloperFeesCalled func() *big.Int - ProcessTransactionFeeCalled func(cost *big.Int, devFee *big.Int, hash []byte) - RevertFeesCalled func(txHashes [][]byte) + CreateBlockStartedCalled func(gasAndFees scheduled.GasAndFees) + GetAccumulatedFeesCalled func() *big.Int + GetDeveloperFeesCalled func() *big.Int + ProcessTransactionFeeCalled func(cost *big.Int, devFee *big.Int, hash []byte) + ProcessTransactionFeeRelayedUserTxCalled func(cost *big.Int, devFee *big.Int, userTxHash []byte, originalTxHash []byte) + RevertFeesCalled func(txHashes [][]byte) } // RevertFees - @@ -52,6 +53,13 @@ func (f *FeeAccumulatorStub) ProcessTransactionFee(cost *big.Int, devFee *big.In } } +// ProcessTransactionFeeRelayedUserTx - +func (f *FeeAccumulatorStub) ProcessTransactionFeeRelayedUserTx(cost *big.Int, devFee *big.Int, userTxHash []byte, originalTxHash []byte) { + if f.ProcessTransactionFeeRelayedUserTxCalled != nil { + f.ProcessTransactionFeeRelayedUserTxCalled(cost, devFee, userTxHash, originalTxHash) + } +} + // IsInterfaceNil - func (f *FeeAccumulatorStub) IsInterfaceNil() bool { return f == nil diff --git a/process/transaction/export_test.go b/process/transaction/export_test.go index b56ebdb4500..5451a5c3f6f 100644 --- a/process/transaction/export_test.go +++ b/process/transaction/export_test.go @@ -60,8 +60,9 @@ func (txProc *txProcessor) ProcessMoveBalanceCostRelayedUserTx( userTx *transaction.Transaction, userScr *smartContractResult.SmartContractResult, userAcc state.UserAccountHandler, + originalTxHash []byte, ) error { - return txProc.processMoveBalanceCostRelayedUserTx(userTx, userScr, userAcc) + return txProc.processMoveBalanceCostRelayedUserTx(userTx, userScr, userAcc, originalTxHash) } func (txProc *txProcessor) ExecuteFailedRelayedTransaction( diff --git a/process/transaction/shardProcess.go b/process/transaction/shardProcess.go index adb575a45d1..882b8595446 100644 --- a/process/transaction/shardProcess.go +++ b/process/transaction/shardProcess.go @@ -211,7 +211,7 @@ func (txProc *txProcessor) ProcessTransaction(tx *transaction.Transaction) (vmco switch txType { case process.MoveBalance: - err = txProc.processMoveBalance(tx, acntSnd, acntDst, dstShardTxType, false) + err = txProc.processMoveBalance(tx, acntSnd, acntDst, dstShardTxType, nil, false) if err != nil { return vmcommon.UserError, txProc.executeAfterFailedMoveBalanceTransaction(tx, err) } @@ -436,6 +436,7 @@ func (txProc *txProcessor) processMoveBalance( tx *transaction.Transaction, acntSrc, acntDst state.UserAccountHandler, destShardTxType process.TransactionType, + originalTxHash []byte, isUserTxOfRelayed bool, ) error { @@ -494,7 +495,11 @@ func (txProc *txProcessor) processMoveBalance( return err } - txProc.txFeeHandler.ProcessTransactionFee(moveBalanceCost, big.NewInt(0), txHash) + if isUserTxOfRelayed { + txProc.txFeeHandler.ProcessTransactionFeeRelayedUserTx(moveBalanceCost, big.NewInt(0), txHash, originalTxHash) + } else { + txProc.txFeeHandler.ProcessTransactionFee(moveBalanceCost, big.NewInt(0), txHash) + } return nil } @@ -717,6 +722,7 @@ func (txProc *txProcessor) processMoveBalanceCostRelayedUserTx( userTx *transaction.Transaction, userScr *smartContractResult.SmartContractResult, userAcc state.UserAccountHandler, + originalTxHash []byte, ) error { moveBalanceGasLimit := txProc.economicsFee.ComputeGasLimit(userTx) moveBalanceUserFee := txProc.economicsFee.ComputeFeeForProcessing(userTx, moveBalanceGasLimit) @@ -726,7 +732,7 @@ func (txProc *txProcessor) processMoveBalanceCostRelayedUserTx( return err } - txProc.txFeeHandler.ProcessTransactionFee(moveBalanceUserFee, big.NewInt(0), userScrHash) + txProc.txFeeHandler.ProcessTransactionFeeRelayedUserTx(moveBalanceUserFee, big.NewInt(0), userScrHash, originalTxHash) return userAcc.SubFromBalance(moveBalanceUserFee) } @@ -766,26 +772,32 @@ func (txProc *txProcessor) processUserTx( return 0, err } + var originalTxHash []byte + originalTxHash, err = core.CalculateHash(txProc.marshalizer, txProc.hasher, originalTx) + if err != nil { + return 0, err + } + returnCode := vmcommon.Ok switch txType { case process.MoveBalance: - err = txProc.processMoveBalance(userTx, acntSnd, acntDst, dstShardTxType, true) + err = txProc.processMoveBalance(userTx, acntSnd, acntDst, dstShardTxType, originalTxHash, true) case process.SCDeployment: - err = txProc.processMoveBalanceCostRelayedUserTx(userTx, scrFromTx, acntSnd) + err = txProc.processMoveBalanceCostRelayedUserTx(userTx, scrFromTx, acntSnd, originalTxHash) if err != nil { break } returnCode, err = txProc.scProcessor.DeploySmartContract(scrFromTx, acntSnd) case process.SCInvoking: - err = txProc.processMoveBalanceCostRelayedUserTx(userTx, scrFromTx, acntSnd) + err = txProc.processMoveBalanceCostRelayedUserTx(userTx, scrFromTx, acntSnd, originalTxHash) if err != nil { break } returnCode, err = txProc.scProcessor.ExecuteSmartContractTransaction(scrFromTx, acntSnd, acntDst) case process.BuiltInFunctionCall: - err = txProc.processMoveBalanceCostRelayedUserTx(userTx, scrFromTx, acntSnd) + err = txProc.processMoveBalanceCostRelayedUserTx(userTx, scrFromTx, acntSnd, originalTxHash) if err != nil { break } diff --git a/process/transaction/shardProcess_test.go b/process/transaction/shardProcess_test.go index c45df730409..53f103682c3 100644 --- a/process/transaction/shardProcess_test.go +++ b/process/transaction/shardProcess_test.go @@ -21,9 +21,9 @@ import ( "github.com/ElrondNetwork/elrond-go/sharding" "github.com/ElrondNetwork/elrond-go/state" "github.com/ElrondNetwork/elrond-go/testscommon" - stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" "github.com/ElrondNetwork/elrond-go/testscommon/epochNotifier" "github.com/ElrondNetwork/elrond-go/testscommon/hashingMocks" + stateMock "github.com/ElrondNetwork/elrond-go/testscommon/state" "github.com/ElrondNetwork/elrond-go/vm" vmcommon "github.com/ElrondNetwork/elrond-vm-common" "github.com/ElrondNetwork/elrond-vm-common/builtInFunctions" @@ -2656,6 +2656,7 @@ func TestTxProcessor_ConsumeMoveBalanceWithUserTx(t *testing.T) { acntSrc, _ := state.NewUserAccount([]byte("address")) acntSrc.Balance = big.NewInt(100) + originalTxHash := []byte("originalTxHash") userTx := &transaction.Transaction{ Nonce: 0, Value: big.NewInt(0), @@ -2663,7 +2664,7 @@ func TestTxProcessor_ConsumeMoveBalanceWithUserTx(t *testing.T) { GasLimit: 100, } - err := execTx.ProcessMoveBalanceCostRelayedUserTx(userTx, &smartContractResult.SmartContractResult{}, acntSrc) + err := execTx.ProcessMoveBalanceCostRelayedUserTx(userTx, &smartContractResult.SmartContractResult{}, acntSrc, originalTxHash) assert.Nil(t, err) assert.Equal(t, acntSrc.Balance, big.NewInt(99)) } diff --git a/testscommon/unsignedTxHandlerStub.go b/testscommon/unsignedTxHandlerStub.go index b33e8ab93ab..42e7b92b65b 100644 --- a/testscommon/unsignedTxHandlerStub.go +++ b/testscommon/unsignedTxHandlerStub.go @@ -9,14 +9,15 @@ import ( // UnsignedTxHandlerStub - type UnsignedTxHandlerStub struct { - CleanProcessedUtxsCalled func() - ProcessTransactionFeeCalled func(cost *big.Int, fee *big.Int, hash []byte) - CreateAllUTxsCalled func() []data.TransactionHandler - VerifyCreatedUTxsCalled func() error - AddTxFeeFromBlockCalled func(tx data.TransactionHandler) - GetAccumulatedFeesCalled func() *big.Int - GetDeveloperFeesCalled func() *big.Int - RevertFeesCalled func(txHashes [][]byte) + CleanProcessedUtxsCalled func() + ProcessTransactionFeeCalled func(cost *big.Int, fee *big.Int, hash []byte) + ProcessTransactionFeeRelayedUserTxCalled func(cost *big.Int, devFee *big.Int, userTxHash []byte, originalTxHash []byte) + CreateAllUTxsCalled func() []data.TransactionHandler + VerifyCreatedUTxsCalled func() error + AddTxFeeFromBlockCalled func(tx data.TransactionHandler) + GetAccumulatedFeesCalled func() *big.Int + GetDeveloperFeesCalled func() *big.Int + RevertFeesCalled func(txHashes [][]byte) } // RevertFees - @@ -73,6 +74,15 @@ func (ut *UnsignedTxHandlerStub) ProcessTransactionFee(cost *big.Int, devFee *bi ut.ProcessTransactionFeeCalled(cost, devFee, txHash) } +// ProcessTransactionFeeRelayedUserTx - +func (ut *UnsignedTxHandlerStub) ProcessTransactionFeeRelayedUserTx(cost *big.Int, devFee *big.Int, userTxHash []byte, originalTxHash []byte) { + if ut.ProcessTransactionFeeRelayedUserTxCalled == nil { + return + } + + ut.ProcessTransactionFeeRelayedUserTxCalled(cost, devFee, userTxHash, originalTxHash) +} + // CreateAllUTxs - func (ut *UnsignedTxHandlerStub) CreateAllUTxs() []data.TransactionHandler { if ut.CreateAllUTxsCalled == nil { From 39253e9b9072ebf85feb65b6e4aebcf79c1235f6 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Mon, 18 Apr 2022 13:57:15 +0300 Subject: [PATCH 05/19] fix esdt total supply endpoint --- .../esdtSupply/esdtSuppliesProcessor_test.go | 224 +++++++++++++++++- dblookupext/esdtSupply/logsProcessor.go | 37 +-- testscommon/genericMocks/storerMock.go | 22 +- 3 files changed, 253 insertions(+), 30 deletions(-) diff --git a/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go b/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go index 0c1ac1ddb44..8da145846cd 100644 --- a/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go +++ b/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go @@ -8,19 +8,23 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core" "github.com/ElrondNetwork/elrond-go-core/data" + "github.com/ElrondNetwork/elrond-go-core/data/block" "github.com/ElrondNetwork/elrond-go-core/data/transaction" "github.com/ElrondNetwork/elrond-go-core/marshal" - "github.com/ElrondNetwork/elrond-go/storage" "github.com/ElrondNetwork/elrond-go/testscommon" + "github.com/ElrondNetwork/elrond-go/testscommon/genericMocks" storageStubs "github.com/ElrondNetwork/elrond-go/testscommon/storage" "github.com/stretchr/testify/require" ) const ( - testNftCreateValue = 10 - testAddQuantityValue = 50 - testBurnValue = 30 + testNftCreateValue = 10 + testAddQuantityValue = 50 + testBurnValue = 30 + testFungibleTokenMint = 100 + testFungibleTokenMint2 = 75 + testFungibleTokenBurn = 75 ) func TestNewSuppliesProcessor(t *testing.T) { @@ -218,14 +222,14 @@ func TestProcessLogsSaveSupplyShouldUpdateSupplyMintedAndBurned(t *testing.T) { require.Equal(t, big.NewInt(testNftCreateValue), supplyEsdt.Minted) case 1: supplyEsdt := getSupplyESDT(marshalizer, data) - require.Equal(t, big.NewInt(testNftCreateValue + testAddQuantityValue), supplyEsdt.Supply) + require.Equal(t, big.NewInt(testNftCreateValue+testAddQuantityValue), supplyEsdt.Supply) require.Equal(t, big.NewInt(0), supplyEsdt.Burned) - require.Equal(t, big.NewInt(testNftCreateValue + testAddQuantityValue), supplyEsdt.Minted) + require.Equal(t, big.NewInt(testNftCreateValue+testAddQuantityValue), supplyEsdt.Minted) case 2: supplyEsdt := getSupplyESDT(marshalizer, data) - require.Equal(t, big.NewInt(testNftCreateValue + testAddQuantityValue - testBurnValue), supplyEsdt.Supply) + require.Equal(t, big.NewInt(testNftCreateValue+testAddQuantityValue-testBurnValue), supplyEsdt.Supply) require.Equal(t, big.NewInt(testBurnValue), supplyEsdt.Burned) - require.Equal(t, big.NewInt(testNftCreateValue + testAddQuantityValue), supplyEsdt.Minted) + require.Equal(t, big.NewInt(testNftCreateValue+testAddQuantityValue), supplyEsdt.Minted) } _ = membDB.Put(key, data) @@ -253,6 +257,210 @@ func TestProcessLogsSaveSupplyShouldUpdateSupplyMintedAndBurned(t *testing.T) { require.Equal(t, 3, numTimesCalled) } +func TestProcessLogs_RevertChangesShouldWorkForRevertingMinting(t *testing.T) { + t.Parallel() + + token := []byte("BRT-1q2w3e") + logsMintNoRevert := []*data.LogData{ + { + TxHash: "txLog0", + LogHandler: &transaction.Log{ + Events: []*transaction.Event{ + { + Identifier: []byte(core.BuiltInFunctionESDTLocalMint), + Topics: [][]byte{ + token, nil, big.NewInt(testFungibleTokenMint).Bytes(), + }, + }, + }, + }, + }, + { + TxHash: "txLog1", + LogHandler: &transaction.Log{ + Events: []*transaction.Event{ + { + Identifier: []byte(core.BuiltInFunctionESDTLocalMint), + Topics: [][]byte{ + token, nil, big.NewInt(testFungibleTokenMint).Bytes(), + }, + }, + }, + }, + }, + } + + mintLogToBeReverted := &transaction.Log{ + Events: []*transaction.Event{ + { + Identifier: []byte(core.BuiltInFunctionESDTLocalMint), + Topics: [][]byte{ + token, nil, big.NewInt(testFungibleTokenMint2).Bytes(), + }, + }, + }, + } + + logsMintRevert := []*data.LogData{ + { + TxHash: "txLog3", + LogHandler: mintLogToBeReverted, + }, + } + + marshalizer := testscommon.MarshalizerMock{} + + logsStorer := genericMocks.NewStorerMockWithErrKeyNotFound("", 0) + mintLogToBeRevertedBytes, err := marshalizer.Marshal(mintLogToBeReverted) + require.NoError(t, err) + err = logsStorer.Put([]byte("txHash3"), mintLogToBeRevertedBytes) + require.NoError(t, err) + + suppliesStorer := genericMocks.NewStorerMockWithErrKeyNotFound("", 0) + + suppliesProc, err := NewSuppliesProcessor(marshalizer, suppliesStorer, logsStorer) + require.Nil(t, err) + + err = suppliesProc.ProcessLogs(6, logsMintNoRevert) + require.Nil(t, err) + checkStoredValues(t, suppliesStorer, token, marshalizer, testFungibleTokenMint*2, testFungibleTokenMint*2, 0) + + err = suppliesProc.ProcessLogs(7, logsMintRevert) + require.Nil(t, err) + checkStoredValues(t, suppliesStorer, token, marshalizer, + testFungibleTokenMint*2+testFungibleTokenMint2, + testFungibleTokenMint*2+testFungibleTokenMint2, 0) + + revertedHeader := block.Header{Nonce: 7} + blockBody := block.Body{ + MiniBlocks: []*block.MiniBlock{ + { + TxHashes: [][]byte{ + []byte("txHash3"), + }, + }, + }, + } + err = suppliesProc.RevertChanges(&revertedHeader, &blockBody) + require.NoError(t, err) + checkStoredValues(t, suppliesStorer, token, marshalizer, + testFungibleTokenMint*2, + testFungibleTokenMint*2, 0) +} + +func TestProcessLogs_RevertChangesShouldWorkForRevertingBurning(t *testing.T) { + t.Parallel() + + token := []byte("BRT-1q2w3e") + logsMintNoRevert := []*data.LogData{ + { + TxHash: "txLog0", + LogHandler: &transaction.Log{ + Events: []*transaction.Event{ + { + Identifier: []byte(core.BuiltInFunctionESDTLocalMint), + Topics: [][]byte{ + token, nil, big.NewInt(testFungibleTokenMint).Bytes(), + }, + }, + }, + }, + }, + { + TxHash: "txLog1", + LogHandler: &transaction.Log{ + Events: []*transaction.Event{ + { + Identifier: []byte(core.BuiltInFunctionESDTLocalMint), + Topics: [][]byte{ + token, nil, big.NewInt(testFungibleTokenMint).Bytes(), + }, + }, + }, + }, + }, + } + + mintLogToBeReverted := &transaction.Log{ + Events: []*transaction.Event{ + { + Identifier: []byte(core.BuiltInFunctionESDTLocalBurn), + Topics: [][]byte{ + token, nil, big.NewInt(testFungibleTokenBurn).Bytes(), + }, + }, + }, + } + + logsMintRevert := []*data.LogData{ + { + TxHash: "txLog3", + LogHandler: mintLogToBeReverted, + }, + } + + marshalizer := testscommon.MarshalizerMock{} + + logsStorer := genericMocks.NewStorerMockWithErrKeyNotFound("", 0) + mintLogToBeRevertedBytes, err := marshalizer.Marshal(mintLogToBeReverted) + require.NoError(t, err) + err = logsStorer.Put([]byte("txHash3"), mintLogToBeRevertedBytes) + require.NoError(t, err) + + suppliesStorer := genericMocks.NewStorerMockWithErrKeyNotFound("", 0) + + suppliesProc, err := NewSuppliesProcessor(marshalizer, suppliesStorer, logsStorer) + require.Nil(t, err) + + err = suppliesProc.ProcessLogs(6, logsMintNoRevert) + require.Nil(t, err) + checkStoredValues(t, suppliesStorer, token, marshalizer, testFungibleTokenMint*2, testFungibleTokenMint*2, 0) + + err = suppliesProc.ProcessLogs(7, logsMintRevert) + require.Nil(t, err) + checkStoredValues(t, + suppliesStorer, + token, + marshalizer, + testFungibleTokenMint*2-testFungibleTokenMint2, + testFungibleTokenMint*2, + testFungibleTokenBurn) + + revertedHeader := block.Header{Nonce: 7} + blockBody := block.Body{ + MiniBlocks: []*block.MiniBlock{ + { + TxHashes: [][]byte{ + []byte("txHash3"), + }, + }, + }, + } + err = suppliesProc.RevertChanges(&revertedHeader, &blockBody) + require.NoError(t, err) + checkStoredValues(t, + suppliesStorer, + token, + marshalizer, + testFungibleTokenMint*2, + testFungibleTokenMint*2, + 0) +} + +func checkStoredValues(t *testing.T, suppliesStorer storage.Storer, token []byte, marshalizer marshal.Marshalizer, supply uint64, minted uint64, burnt uint64) { + storedSupplyBytes, err := suppliesStorer.Get(token) + require.NoError(t, err) + + var recoveredSupply SupplyESDT + err = marshalizer.Unmarshal(&recoveredSupply, storedSupplyBytes) + require.NoError(t, err) + require.NotNil(t, recoveredSupply) + + require.Equal(t, supply, recoveredSupply.Supply.Uint64()) + require.Equal(t, minted, recoveredSupply.Minted.Uint64()) + require.Equal(t, burnt, recoveredSupply.Burned.Uint64()) +} + func getSupplyESDT(marshalizer marshal.Marshalizer, data []byte) SupplyESDT { var supplyESDT SupplyESDT _ = marshalizer.Unmarshal(&supplyESDT, data) diff --git a/dblookupext/esdtSupply/logsProcessor.go b/dblookupext/esdtSupply/logsProcessor.go index d1d00a7d5cc..33fdf1bc9f6 100644 --- a/dblookupext/esdtSupply/logsProcessor.go +++ b/dblookupext/esdtSupply/logsProcessor.go @@ -148,28 +148,29 @@ func (lp *logsProcessor) processEvent(txLog *transaction.Event, supplies map[str func (lp *logsProcessor) updateTokenSupply(tokenSupply *SupplyESDT, valueFromEvent *big.Int, eventIdentifier string, isRevert bool) { isBurnOP := eventIdentifier == core.BuiltInFunctionESDTLocalBurn || eventIdentifier == core.BuiltInFunctionESDTNFTBurn || eventIdentifier == core.BuiltInFunctionESDTWipe - xorBooleanVariable := isBurnOP != isRevert - // need this because - // negValue | isRevert => res - // false | false => false - // false | true => true - // true | true => false - // true | false => true - bigValue := big.NewInt(0).Set(valueFromEvent) - if xorBooleanVariable { - bigValue = big.NewInt(0).Neg(valueFromEvent) - } + isMintOP := eventIdentifier == core.BuiltInFunctionESDTNFTAddQuantity || eventIdentifier == core.BuiltInFunctionESDTLocalMint || + eventIdentifier == core.BuiltInFunctionESDTNFTCreate + + negativeValueFromEvent := big.NewInt(0).Neg(valueFromEvent) switch { - case isBurnOP: - tokenSupply.Burned.Add(tokenSupply.Burned, valueFromEvent) - case eventIdentifier == core.BuiltInFunctionESDTNFTAddQuantity || - eventIdentifier == core.BuiltInFunctionESDTLocalMint || - eventIdentifier == core.BuiltInFunctionESDTNFTCreate: + case isMintOP && !isRevert: + // normal processing mint - add to supply and add to minted tokenSupply.Minted.Add(tokenSupply.Minted, valueFromEvent) + tokenSupply.Supply.Add(tokenSupply.Supply, valueFromEvent) + case isMintOP && isRevert: + // reverted mint - subtract from supply and subtract from minted + tokenSupply.Minted.Add(tokenSupply.Minted, negativeValueFromEvent) + tokenSupply.Supply.Add(tokenSupply.Supply, negativeValueFromEvent) + case isBurnOP && !isRevert: + // normal processing burn - subtract from supply and add to burn + tokenSupply.Burned.Add(tokenSupply.Burned, valueFromEvent) + tokenSupply.Supply.Add(tokenSupply.Supply, negativeValueFromEvent) + case isBurnOP && isRevert: + // reverted burn - subtract from burned and add to supply + tokenSupply.Burned.Add(tokenSupply.Burned, negativeValueFromEvent) + tokenSupply.Supply.Add(tokenSupply.Supply, valueFromEvent) } - - tokenSupply.Supply.Add(tokenSupply.Supply, bigValue) } func (lp *logsProcessor) getESDTSupply(tokenIdentifier []byte) (*SupplyESDT, error) { diff --git a/testscommon/genericMocks/storerMock.go b/testscommon/genericMocks/storerMock.go index 68f1535a4d7..7bbdd52b178 100644 --- a/testscommon/genericMocks/storerMock.go +++ b/testscommon/genericMocks/storerMock.go @@ -9,14 +9,16 @@ import ( "github.com/ElrondNetwork/elrond-go-core/core/atomic" "github.com/ElrondNetwork/elrond-go-core/core/container" "github.com/ElrondNetwork/elrond-go-core/marshal" + "github.com/ElrondNetwork/elrond-go/storage" ) // StorerMock - type StorerMock struct { - mutex sync.RWMutex - Name string - DataByEpoch map[uint32]*container.MutexMap - currentEpoch atomic.Uint32 + mutex sync.RWMutex + Name string + DataByEpoch map[uint32]*container.MutexMap + shouldReturnErrKeyNotFound bool + currentEpoch atomic.Uint32 } // NewStorerMock - @@ -30,6 +32,14 @@ func NewStorerMock(name string, currentEpoch uint32) *StorerMock { return sm } +// NewStorerMockWithErrKeyNotFound - +func NewStorerMockWithErrKeyNotFound(name string, currentEpoch uint32) *StorerMock { + sm := NewStorerMock(name, currentEpoch) + sm.shouldReturnErrKeyNotFound = true + + return sm +} + // SetCurrentEpoch - func (sm *StorerMock) SetCurrentEpoch(epoch uint32) { sm.currentEpoch.Set(epoch) @@ -210,5 +220,9 @@ func (sm *StorerMock) IsInterfaceNil() bool { } func (sm *StorerMock) newErrNotFound(key []byte, epoch uint32) error { + if sm.shouldReturnErrKeyNotFound { + return storage.ErrKeyNotFound + } + return fmt.Errorf("StorerMock: not found in %s: key = %s, epoch = %d", sm.Name, hex.EncodeToString(key), epoch) } From 67fcbaaa577570d4cf296e20243aceafda9e6ca3 Mon Sep 17 00:00:00 2001 From: AdoAdoAdo Date: Mon, 18 Apr 2022 14:12:19 +0300 Subject: [PATCH 06/19] add unit tests for user txs fees revert --- process/block/postprocess/feeHandler.go | 25 +++---- process/block/postprocess/feeHandler_test.go | 68 ++++++++++++++++++++ 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/process/block/postprocess/feeHandler.go b/process/block/postprocess/feeHandler.go index 6da0ad76157..389b178a055 100644 --- a/process/block/postprocess/feeHandler.go +++ b/process/block/postprocess/feeHandler.go @@ -85,6 +85,19 @@ func (f *feeHandler) ProcessTransactionFeeRelayedUserTx(cost *big.Int, devFee *b f.mut.Unlock() } +// RevertFees reverts the accumulated fees for txHashes +func (f *feeHandler) RevertFees(txHashes [][]byte) { + f.mut.Lock() + defer f.mut.Unlock() + + for _, txHash := range txHashes { + f.revertFeesForDependentTxHash(txHash) + } + for _, txHash := range txHashes { + f.revertFee(txHash) + } +} + func (f *feeHandler) linkRelayedUserTxToOriginalTx(userTxHash []byte, originalTxHash []byte) { f.mapDependentHashes[string(originalTxHash)] = userTxHash } @@ -95,17 +108,7 @@ func (f *feeHandler) revertFeesForDependentTxHash(txHash []byte) { return } f.revertFee(linkedTxHash) -} - -// RevertFees reverts the accumulated fees for txHashes -func (f *feeHandler) RevertFees(txHashes [][]byte) { - f.mut.Lock() - defer f.mut.Unlock() - - for _, txHash := range txHashes { - f.revertFeesForDependentTxHash(txHash) - f.revertFee(txHash) - } + delete(f.mapDependentHashes, string(txHash)) } func (f *feeHandler) revertFee(txHash []byte) { diff --git a/process/block/postprocess/feeHandler_test.go b/process/block/postprocess/feeHandler_test.go index 1f86fde5bdb..91ed6c00bce 100644 --- a/process/block/postprocess/feeHandler_test.go +++ b/process/block/postprocess/feeHandler_test.go @@ -86,6 +86,74 @@ func TestFeeHandler_RevertFees(t *testing.T) { require.Equal(t, big.NewInt(101), devFees) } +func TestFeeHandler_CompleteRevertFeesUserTxs(t *testing.T) { + t.Parallel() + + feeHandler, _ := postprocess.NewFeeAccumulator() + userTxHashes := [][]byte{[]byte("txHash1"), []byte("txHash2"), []byte("txHash3")} + originalTxHashes := [][]byte{[]byte("origTxHash1"), []byte("origTxHash2"), []byte("origTxHash3")} + + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(1000), big.NewInt(100), userTxHashes[0], originalTxHashes[0]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(100), big.NewInt(10), userTxHashes[1], originalTxHashes[1]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(10), big.NewInt(1), userTxHashes[2], originalTxHashes[2]) + + feeHandler.RevertFees(originalTxHashes) + + accFees := feeHandler.GetAccumulatedFees() + devFees := feeHandler.GetDeveloperFees() + require.Equal(t, big.NewInt(0), accFees) + require.Equal(t, big.NewInt(0), devFees) +} + +func TestFeeHandler_PartialRevertFeesUserTxs(t *testing.T) { + t.Parallel() + userTxHashes := [][]byte{[]byte("txHash1"), []byte("txHash2"), []byte("txHash3")} + originalTxHashes := [][]byte{[]byte("origTxHash1"), []byte("origTxHash2"), []byte("origTxHash3"), []byte("userTxHash4")} + + t.Run("revert partial originalTxs", func(t *testing.T) { + feeHandler, _ := postprocess.NewFeeAccumulator() + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(1000), big.NewInt(100), userTxHashes[0], originalTxHashes[0]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(100), big.NewInt(10), userTxHashes[1], originalTxHashes[1]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(10), big.NewInt(1), userTxHashes[2], originalTxHashes[2]) + feeHandler.ProcessTransactionFee(big.NewInt(2000), big.NewInt(200), originalTxHashes[3]) + + feeHandler.RevertFees(originalTxHashes[:3]) + + accFees := feeHandler.GetAccumulatedFees() + devFees := feeHandler.GetDeveloperFees() + require.Equal(t, big.NewInt(2000), accFees) + require.Equal(t, big.NewInt(200), devFees) + }) + t.Run("revert all userTxs", func(t *testing.T) { + feeHandler, _ := postprocess.NewFeeAccumulator() + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(1000), big.NewInt(100), userTxHashes[0], originalTxHashes[0]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(100), big.NewInt(10), userTxHashes[1], originalTxHashes[1]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(10), big.NewInt(1), userTxHashes[2], originalTxHashes[2]) + feeHandler.ProcessTransactionFee(big.NewInt(2000), big.NewInt(200), originalTxHashes[3]) + + feeHandler.RevertFees(userTxHashes) + + accFees := feeHandler.GetAccumulatedFees() + devFees := feeHandler.GetDeveloperFees() + require.Equal(t, big.NewInt(2000), accFees) + require.Equal(t, big.NewInt(200), devFees) + }) + t.Run("revert partial userTxs", func(t *testing.T) { + feeHandler, _ := postprocess.NewFeeAccumulator() + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(1000), big.NewInt(100), userTxHashes[0], originalTxHashes[0]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(100), big.NewInt(10), userTxHashes[1], originalTxHashes[1]) + feeHandler.ProcessTransactionFeeRelayedUserTx(big.NewInt(10), big.NewInt(1), userTxHashes[2], originalTxHashes[2]) + feeHandler.ProcessTransactionFee(big.NewInt(2000), big.NewInt(200), originalTxHashes[3]) + + feeHandler.RevertFees(userTxHashes[:2]) + + accFees := feeHandler.GetAccumulatedFees() + devFees := feeHandler.GetDeveloperFees() + require.Equal(t, big.NewInt(2010), accFees) + require.Equal(t, big.NewInt(201), devFees) + }) +} + func TestFeeHandler_IsInterfaceNil(t *testing.T) { t.Parallel() From 617f9a8875943c9299f550e54b7ea0277213c96a Mon Sep 17 00:00:00 2001 From: AdoAdoAdo Date: Mon, 18 Apr 2022 14:17:02 +0300 Subject: [PATCH 07/19] process: simplify method --- process/block/postprocess/feeHandler.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/process/block/postprocess/feeHandler.go b/process/block/postprocess/feeHandler.go index 389b178a055..75bed3683cf 100644 --- a/process/block/postprocess/feeHandler.go +++ b/process/block/postprocess/feeHandler.go @@ -92,8 +92,6 @@ func (f *feeHandler) RevertFees(txHashes [][]byte) { for _, txHash := range txHashes { f.revertFeesForDependentTxHash(txHash) - } - for _, txHash := range txHashes { f.revertFee(txHash) } } From c4eecc36d46bde2aeef05a1bf1bbcaac64a7dfe2 Mon Sep 17 00:00:00 2001 From: robertsasu Date: Tue, 19 Apr 2022 14:07:51 +0300 Subject: [PATCH 08/19] added internal vm errors as log added txHash in executing transaction print --- process/common.go | 2 ++ process/rewardTransaction/process.go | 1 + process/smartContract/process.go | 20 ++++++++++++++++++-- process/transaction/metaProcess.go | 1 + process/transaction/shardProcess.go | 6 ++++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/process/common.go b/process/common.go index 97b7790fb69..42dd884b38d 100644 --- a/process/common.go +++ b/process/common.go @@ -607,6 +607,7 @@ func DisplayProcessTxDetails( message string, accountHandler vmcommon.AccountHandler, txHandler data.TransactionHandler, + txHash []byte, addressPubkeyConverter core.PubkeyConverter, ) { if !check.IfNil(accountHandler) { @@ -637,6 +638,7 @@ func DisplayProcessTxDetails( } log.Trace("executing transaction", + "txHash", txHash, "nonce", txHandler.GetNonce(), "value", txHandler.GetValue(), "gas limit", txHandler.GetGasLimit(), diff --git a/process/rewardTransaction/process.go b/process/rewardTransaction/process.go index 8ef352c1fe2..099124ec611 100644 --- a/process/rewardTransaction/process.go +++ b/process/rewardTransaction/process.go @@ -87,6 +87,7 @@ func (rtp *rewardTxProcessor) ProcessRewardTransaction(rTx *rewardTx.RewardTx) e "ProcessRewardTransaction: receiver account details", accHandler, rTx, + nil, rtp.pubkeyConv, ) diff --git a/process/smartContract/process.go b/process/smartContract/process.go index a3b3ab2bb36..be3e710cedf 100644 --- a/process/smartContract/process.go +++ b/process/smartContract/process.go @@ -459,7 +459,7 @@ func (sc *scProcessor) executeSmartContractCall( vmOutput.GasRemaining += vmInput.GasLocked if vmOutput.ReturnCode != vmcommon.Ok { - return userErrorVmOutput, sc.ProcessIfError(acntSnd, txHash, tx, vmOutput.ReturnCode.String(), []byte(vmOutput.ReturnMessage), snapshot, vmInput.GasLocked) + return userErrorVmOutput, sc.processIfErrorWithAddedLogs(acntSnd, txHash, tx, vmOutput.ReturnCode.String(), []byte(vmOutput.ReturnMessage), snapshot, vmInput.GasLocked, vmOutput.Logs) } acntSnd, err = sc.reloadLocalAccount(acntSnd) // nolint @@ -1352,6 +1352,18 @@ func (sc *scProcessor) ProcessIfError( returnMessage []byte, snapshot int, gasLocked uint64, +) error { + return sc.processIfErrorWithAddedLogs(acntSnd, txHash, tx, returnCode, returnMessage, snapshot, gasLocked, nil) +} + +func (sc *scProcessor) processIfErrorWithAddedLogs(acntSnd state.UserAccountHandler, + txHash []byte, + tx data.TransactionHandler, + returnCode string, + returnMessage []byte, + snapshot int, + gasLocked uint64, + internalVMLogs []*vmcommon.LogEntry, ) error { sc.vmOutputCacher.Put(txHash, &vmcommon.VMOutput{ ReturnCode: vmcommon.SimulateFailed, @@ -1400,6 +1412,9 @@ func (sc *scProcessor) ProcessIfError( if relayerLog != nil { processIfErrorLogs = append(processIfErrorLogs, relayerLog) } + if len(internalVMLogs) > 0 { + processIfErrorLogs = append(processIfErrorLogs, internalVMLogs...) + } logsTxHash := sc.getOriginalTxHashIfIntraShardRelayedSCR(tx, txHash) ignorableError := sc.txLogsProcessor.SaveLog(logsTxHash, tx, processIfErrorLogs) @@ -1710,7 +1725,7 @@ func (sc *scProcessor) doDeploySmartContract( } vmOutput.GasRemaining += vmInput.GasLocked if vmOutput.ReturnCode != vmcommon.Ok { - return vmcommon.UserError, sc.ProcessIfError(acntSnd, txHash, tx, vmOutput.ReturnCode.String(), []byte(vmOutput.ReturnMessage), snapshot, vmInput.GasLocked) + return vmcommon.UserError, sc.processIfErrorWithAddedLogs(acntSnd, txHash, tx, vmOutput.ReturnCode.String(), []byte(vmOutput.ReturnMessage), snapshot, vmInput.GasLocked, vmOutput.Logs) } err = sc.gasConsumedChecks(tx, vmInput.GasProvided, vmInput.GasLocked, vmOutput) @@ -2683,6 +2698,7 @@ func (sc *scProcessor) ProcessSmartContractResult(scr *smartContractResult.Smart "ProcessSmartContractResult: receiver account details", dstAcc, scr, + txHash, sc.pubkeyConv, ) diff --git a/process/transaction/metaProcess.go b/process/transaction/metaProcess.go index 277ac3d7172..ec6d15f3b5f 100644 --- a/process/transaction/metaProcess.go +++ b/process/transaction/metaProcess.go @@ -112,6 +112,7 @@ func (txProc *metaTxProcessor) ProcessTransaction(tx *transaction.Transaction) ( "ProcessTransaction: sender account details", acntSnd, tx, + txHash, txProc.pubkeyConv, ) diff --git a/process/transaction/shardProcess.go b/process/transaction/shardProcess.go index adb575a45d1..e8a92822695 100644 --- a/process/transaction/shardProcess.go +++ b/process/transaction/shardProcess.go @@ -175,10 +175,16 @@ func (txProc *txProcessor) ProcessTransaction(tx *transaction.Transaction) (vmco return 0, err } + txHash, err := core.CalculateHash(txProc.marshalizer, txProc.hasher, tx) + if err != nil { + return 0, err + } + process.DisplayProcessTxDetails( "ProcessTransaction: sender account details", acntSnd, tx, + txHash, txProc.pubkeyConv, ) From c90ea393c058c9a1cba535ad87b2f72d3b3a3cb4 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Mon, 18 Apr 2022 16:05:27 +0300 Subject: [PATCH 09/19] don't display synchronized when termui starts --- .../view/termuic/termuiRenders/widgetsRender.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go index d4ee7310631..95ec6b0ce3a 100644 --- a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go +++ b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go @@ -11,9 +11,12 @@ import ( "github.com/gizak/termui/v3/widgets" ) -const statusSyncing = "currently syncing" -const statusSynchronized = "synchronized" -const invalidKey = "invalid key" +const ( + statusSyncing = "currently syncing" + statusSynchronized = "synchronized" + statusNotApplicable = "N/A" + invalidKey = "invalid key" +) // WidgetsRender will define termui widgets that need to display a termui console type WidgetsRender struct { @@ -211,6 +214,8 @@ func (wr *WidgetsRender) prepareChainInfo(numMillisecondsRefreshTime int) { blocksPerSecond := wr.presenter.CalculateSynchronizationSpeed(numMillisecondsRefreshTime) blocksPerSecondMessage = fmt.Sprintf("%d blocks/sec", blocksPerSecond) + case synchronizedRound == currentRound && currentRound == 0: + syncingStr = statusNotApplicable default: syncingStr = statusSynchronized } From 332e3f0bdfd4a347e16a62648e50d7e7add05489 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Mon, 18 Apr 2022 17:30:59 +0300 Subject: [PATCH 10/19] added redundancy info in termui --- cmd/termui/presenter/instanceInfoGetters.go | 10 ++++++++ cmd/termui/view/interface.go | 2 ++ .../termuic/termuiRenders/widgetsRender.go | 23 ++++++++++++++++++- common/constants.go | 13 +++++++++-- common/converters.go | 9 ++++++++ common/converters_test.go | 16 +++++++++++++ consensus/spos/worker.go | 1 + node/nodeRunner.go | 2 ++ 8 files changed, 73 insertions(+), 3 deletions(-) diff --git a/cmd/termui/presenter/instanceInfoGetters.go b/cmd/termui/presenter/instanceInfoGetters.go index 78ee41a52e9..8459d0b51dc 100644 --- a/cmd/termui/presenter/instanceInfoGetters.go +++ b/cmd/termui/presenter/instanceInfoGetters.go @@ -34,6 +34,16 @@ func (psh *PresenterStatusHandler) GetPublicKeyBlockSign() string { return psh.getFromCacheAsString(common.MetricPublicKeyBlockSign) } +// GetRedundancyLevel will return the redundancy level of the node +func (psh *PresenterStatusHandler) GetRedundancyLevel() uint64 { + return psh.getFromCacheAsUint64(common.MetricRedundancyLevel) +} + +// GetRedundancyIsMainActive will return the info about redundancy main machine +func (psh *PresenterStatusHandler) GetRedundancyIsMainActive() string { + return psh.getFromCacheAsString(common.MetricRedundancyIsMainActive) +} + // GetShardId will return shard Id of node func (psh *PresenterStatusHandler) GetShardId() uint64 { return psh.getFromCacheAsUint64(common.MetricShardId) diff --git a/cmd/termui/view/interface.go b/cmd/termui/view/interface.go index f67a3a1f73a..3c331b74750 100644 --- a/cmd/termui/view/interface.go +++ b/cmd/termui/view/interface.go @@ -5,6 +5,8 @@ type Presenter interface { GetAppVersion() string GetNodeName() string GetPublicKeyBlockSign() string + GetRedundancyLevel() uint64 + GetRedundancyIsMainActive() string GetShardId() uint64 GetNodeType() string GetPeerType() string diff --git a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go index 95ec6b0ce3a..171b82b5176 100644 --- a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go +++ b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go @@ -165,6 +165,8 @@ func (wr *WidgetsRender) prepareInstanceInfo() { countAcceptedBlocks := wr.presenter.GetCountAcceptedBlocks() rows[4] = []string{fmt.Sprintf("Blocks proposed: %d | Blocks accepted: %d", countLeader, countAcceptedBlocks)} + rows[5] = []string{computeRedundancyStr(wr.presenter.GetRedundancyLevel(), wr.presenter.GetRedundancyIsMainActive())} + // TODO: repair the rewards estimation or replace these 2 rows with rating details //switch instanceType { //case string(common.NodeTypeValidator): @@ -188,7 +190,6 @@ func (wr *WidgetsRender) prepareInstanceInfo() { // rows[6] = []string{""} //} - rows[5] = []string{""} rows[6] = []string{""} wr.instanceInfo.Title = "Elrond instance info" @@ -268,6 +269,26 @@ func (wr *WidgetsRender) prepareChainInfo(numMillisecondsRefreshTime int) { wr.chainInfo.Rows = rows } +func computeRedundancyStr(redundancyLevel uint64, redundancyIsMainActive string) string { + if redundancyIsMainActive == statusNotApplicable { + return "" + } + + redundancyStr := "Redundancy: " + if redundancyLevel < 0 { + redundancyStr += "inactive" + } else { + if redundancyLevel == 0 { + redundancyStr += "Main machine" + } else { + redundancyStr += fmt.Sprintf("back-up #%d", redundancyLevel) + redundancyStr += fmt.Sprintf(" (is main active: %s)", redundancyIsMainActive) + } + } + + return redundancyStr +} + func (wr *WidgetsRender) prepareBlockInfo() { //7 rows and one column numRows := 8 diff --git a/common/constants.go b/common/constants.go index a457b5013f1..10f12d09227 100644 --- a/common/constants.go +++ b/common/constants.go @@ -276,16 +276,25 @@ const MetricRoundsPassedInCurrentEpoch = "erd_rounds_passed_in_current_epoch" // MetricNoncesPassedInCurrentEpoch is the metric that tells the number of nonces passed in current epoch const MetricNoncesPassedInCurrentEpoch = "erd_nonces_passed_in_current_epoch" -//MetricReceivedProposedBlock is the metric that specify the moment in the round when the received block has reached the +//MetricReceivedProposedBlock is the metric that specifies the moment in the round when the received block has reached the //current node. The value is provided in percent (0 meaning it has been received just after the round started and //100 meaning that the block has been received in the last moment of the round) const MetricReceivedProposedBlock = "erd_consensus_received_proposed_block" -//MetricCreatedProposedBlock is the metric that specify the percent of the block subround used for header and body +//MetricCreatedProposedBlock is the metric that specifies the percent of the block subround used for header and body //creation (0 meaning that the block was created in no-time and 100 meaning that the block creation used all the //subround spare duration) const MetricCreatedProposedBlock = "erd_consensus_created_proposed_block" +// MetricRedundancyLevel is the metric that specifies the redundancy level of the current node +const MetricRedundancyLevel = "erd_redundancy_level" + +// MetricRedundancyMainActive is the metrics that specifies data about the redundancy main machine +const MetricRedundancyIsMainActive = "erd_redundancy_is_main_active" + +// MetricValueNA represents the value to be used when a metric is not available/applicable +const MetricValueNA = "N/A" + //MetricProcessedProposedBlock is the metric that specify the percent of the block subround used for header and body //processing (0 meaning that the block was processed in no-time and 100 meaning that the block processing used all the //subround spare duration) diff --git a/common/converters.go b/common/converters.go index d37cb9d49bc..898bb016766 100644 --- a/common/converters.go +++ b/common/converters.go @@ -46,3 +46,12 @@ func AssignShardForPubKeyWhenNotSpecified(pubKey []byte, numShards uint32) uint3 return randomShardID } + +// BoolToString converts a boolean value into it's string representation +func BoolToString(value bool) string { + if value { + return "true" + } + + return "false" +} diff --git a/common/converters_test.go b/common/converters_test.go index cda64640239..c1c7a36a167 100644 --- a/common/converters_test.go +++ b/common/converters_test.go @@ -329,3 +329,19 @@ func TestShardAssignment(t *testing.T) { fmt.Printf("Shard %d:\n\t\t%d accounts\n", sh, cnt) } } + +func TestBoolToString(t *testing.T) { + t.Parallel() + + t.Run("true", func(t *testing.T) { + t.Parallel() + + require.Equal(t, "true", common.BoolToString(true)) + }) + + t.Run("false", func(t *testing.T) { + t.Parallel() + + require.Equal(t, "false", common.BoolToString(false)) + }) +} diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index 7f9118313c8..fb420467c8d 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -513,6 +513,7 @@ func (wrk *Worker) processReceivedHeaderMetric(cnsDta *consensus.Message) { sinceRoundStart := time.Since(wrk.roundHandler.TimeStamp()) percent := sinceRoundStart * 100 / wrk.roundHandler.TimeDuration() wrk.appStatusHandler.SetUInt64Value(common.MetricReceivedProposedBlock, uint64(percent)) + wrk.appStatusHandler.SetStringValue(common.MetricRedundancyIsMainActive, common.BoolToString(wrk.nodeRedundancyHandler.IsMainMachineActive())) } func (wrk *Worker) checkSelfState(cnsDta *consensus.Message) error { diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 1f4a8505317..678b7261a33 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -587,6 +587,8 @@ func (nr *nodeRunner) createMetrics( } metrics.SaveStringMetric(coreComponents.StatusHandler(), common.MetricNodeDisplayName, nr.configs.PreferencesConfig.Preferences.NodeDisplayName) + metrics.SaveUint64Metric(coreComponents.StatusHandler(), common.MetricRedundancyLevel, uint64(nr.configs.PreferencesConfig.Preferences.RedundancyLevel)) + metrics.SaveStringMetric(coreComponents.StatusHandler(), common.MetricRedundancyIsMainActive, common.MetricValueNA) metrics.SaveStringMetric(coreComponents.StatusHandler(), common.MetricChainId, coreComponents.ChainID()) metrics.SaveUint64Metric(coreComponents.StatusHandler(), common.MetricGasPerDataByte, coreComponents.EconomicsData().GasPerDataByte()) metrics.SaveUint64Metric(coreComponents.StatusHandler(), common.MetricMinGasPrice, coreComponents.EconomicsData().MinGasPrice()) From a4aebd1cecd89f955f509691193a580fa8ad5bf5 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Tue, 19 Apr 2022 16:16:46 +0300 Subject: [PATCH 11/19] fixes after review --- common/converters.go | 9 --------- common/converters_test.go | 16 ---------------- consensus/spos/worker.go | 3 ++- .../esdtSupply/esdtSuppliesProcessor_test.go | 4 ++-- dblookupext/esdtSupply/logsProcessor.go | 12 ++++++------ 5 files changed, 10 insertions(+), 34 deletions(-) diff --git a/common/converters.go b/common/converters.go index 898bb016766..d37cb9d49bc 100644 --- a/common/converters.go +++ b/common/converters.go @@ -46,12 +46,3 @@ func AssignShardForPubKeyWhenNotSpecified(pubKey []byte, numShards uint32) uint3 return randomShardID } - -// BoolToString converts a boolean value into it's string representation -func BoolToString(value bool) string { - if value { - return "true" - } - - return "false" -} diff --git a/common/converters_test.go b/common/converters_test.go index c1c7a36a167..cda64640239 100644 --- a/common/converters_test.go +++ b/common/converters_test.go @@ -329,19 +329,3 @@ func TestShardAssignment(t *testing.T) { fmt.Printf("Shard %d:\n\t\t%d accounts\n", sh, cnt) } } - -func TestBoolToString(t *testing.T) { - t.Parallel() - - t.Run("true", func(t *testing.T) { - t.Parallel() - - require.Equal(t, "true", common.BoolToString(true)) - }) - - t.Run("false", func(t *testing.T) { - t.Parallel() - - require.Equal(t, "false", common.BoolToString(false)) - }) -} diff --git a/consensus/spos/worker.go b/consensus/spos/worker.go index fb420467c8d..788cf447ffc 100644 --- a/consensus/spos/worker.go +++ b/consensus/spos/worker.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "fmt" + "strconv" "sync" "time" @@ -513,7 +514,7 @@ func (wrk *Worker) processReceivedHeaderMetric(cnsDta *consensus.Message) { sinceRoundStart := time.Since(wrk.roundHandler.TimeStamp()) percent := sinceRoundStart * 100 / wrk.roundHandler.TimeDuration() wrk.appStatusHandler.SetUInt64Value(common.MetricReceivedProposedBlock, uint64(percent)) - wrk.appStatusHandler.SetStringValue(common.MetricRedundancyIsMainActive, common.BoolToString(wrk.nodeRedundancyHandler.IsMainMachineActive())) + wrk.appStatusHandler.SetStringValue(common.MetricRedundancyIsMainActive, strconv.FormatBool(wrk.nodeRedundancyHandler.IsMainMachineActive())) } func (wrk *Worker) checkSelfState(cnsDta *consensus.Message) error { diff --git a/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go b/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go index 8da145846cd..10093b97106 100644 --- a/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go +++ b/dblookupext/esdtSupply/esdtSuppliesProcessor_test.go @@ -24,7 +24,7 @@ const ( testBurnValue = 30 testFungibleTokenMint = 100 testFungibleTokenMint2 = 75 - testFungibleTokenBurn = 75 + testFungibleTokenBurn = 25 ) func TestNewSuppliesProcessor(t *testing.T) { @@ -422,7 +422,7 @@ func TestProcessLogs_RevertChangesShouldWorkForRevertingBurning(t *testing.T) { suppliesStorer, token, marshalizer, - testFungibleTokenMint*2-testFungibleTokenMint2, + testFungibleTokenMint*2-testFungibleTokenBurn, testFungibleTokenMint*2, testFungibleTokenBurn) diff --git a/dblookupext/esdtSupply/logsProcessor.go b/dblookupext/esdtSupply/logsProcessor.go index 33fdf1bc9f6..1b79cf4183e 100644 --- a/dblookupext/esdtSupply/logsProcessor.go +++ b/dblookupext/esdtSupply/logsProcessor.go @@ -146,27 +146,27 @@ func (lp *logsProcessor) processEvent(txLog *transaction.Event, supplies map[str } func (lp *logsProcessor) updateTokenSupply(tokenSupply *SupplyESDT, valueFromEvent *big.Int, eventIdentifier string, isRevert bool) { - isBurnOP := eventIdentifier == core.BuiltInFunctionESDTLocalBurn || eventIdentifier == core.BuiltInFunctionESDTNFTBurn || + isBurnOp := eventIdentifier == core.BuiltInFunctionESDTLocalBurn || eventIdentifier == core.BuiltInFunctionESDTNFTBurn || eventIdentifier == core.BuiltInFunctionESDTWipe - isMintOP := eventIdentifier == core.BuiltInFunctionESDTNFTAddQuantity || eventIdentifier == core.BuiltInFunctionESDTLocalMint || + isMintOp := eventIdentifier == core.BuiltInFunctionESDTNFTAddQuantity || eventIdentifier == core.BuiltInFunctionESDTLocalMint || eventIdentifier == core.BuiltInFunctionESDTNFTCreate negativeValueFromEvent := big.NewInt(0).Neg(valueFromEvent) switch { - case isMintOP && !isRevert: + case isMintOp && !isRevert: // normal processing mint - add to supply and add to minted tokenSupply.Minted.Add(tokenSupply.Minted, valueFromEvent) tokenSupply.Supply.Add(tokenSupply.Supply, valueFromEvent) - case isMintOP && isRevert: + case isMintOp && isRevert: // reverted mint - subtract from supply and subtract from minted tokenSupply.Minted.Add(tokenSupply.Minted, negativeValueFromEvent) tokenSupply.Supply.Add(tokenSupply.Supply, negativeValueFromEvent) - case isBurnOP && !isRevert: + case isBurnOp && !isRevert: // normal processing burn - subtract from supply and add to burn tokenSupply.Burned.Add(tokenSupply.Burned, valueFromEvent) tokenSupply.Supply.Add(tokenSupply.Supply, negativeValueFromEvent) - case isBurnOP && isRevert: + case isBurnOp && isRevert: // reverted burn - subtract from burned and add to supply tokenSupply.Burned.Add(tokenSupply.Burned, negativeValueFromEvent) tokenSupply.Supply.Add(tokenSupply.Supply, valueFromEvent) From 2a195a53dc5ca39b2c643c84e41c15c9e47a2c32 Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Tue, 19 Apr 2022 20:29:34 +0300 Subject: [PATCH 12/19] fix after review --- cmd/termui/presenter/instanceInfoGetters.go | 48 ------------- .../presenter/instanceInfoGetters_test.go | 71 ------------------- cmd/termui/view/interface.go | 4 -- .../termuic/termuiRenders/widgetsRender.go | 28 +------- 4 files changed, 2 insertions(+), 149 deletions(-) diff --git a/cmd/termui/presenter/instanceInfoGetters.go b/cmd/termui/presenter/instanceInfoGetters.go index 8459d0b51dc..37e80a24dba 100644 --- a/cmd/termui/presenter/instanceInfoGetters.go +++ b/cmd/termui/presenter/instanceInfoGetters.go @@ -1,14 +1,11 @@ package presenter import ( - "math/big" "strings" "github.com/ElrondNetwork/elrond-go/common" ) -const precisionRewards = 2 - // GetAppVersion will return application version func (psh *PresenterStatusHandler) GetAppVersion() string { return psh.getFromCacheAsString(common.MetricAppVersion) @@ -91,48 +88,3 @@ func (psh *PresenterStatusHandler) GetNodeName() string { return nodeName } - -// GetTotalRewardsValue will return total value of rewards and how the rewards were increased on every second -// Rewards estimation will be equal with : -// numSignedBlocks * denomination * Rewards -func (psh *PresenterStatusHandler) GetTotalRewardsValue() (string, string) { - numSignedBlocks := psh.getFromCacheAsUint64(common.MetricCountConsensusAcceptedBlocks) - rewardsInErd := psh.computeRewardsInErd() - - totalRewardsFloat := big.NewFloat(float64(numSignedBlocks)) - totalRewardsFloat.Mul(totalRewardsFloat, rewardsInErd) - difRewards := big.NewFloat(0).Sub(totalRewardsFloat, psh.totalRewardsOld) - - defer func() { - psh.totalRewardsOld = totalRewardsFloat - }() - - return psh.totalRewardsOld.Text('f', precisionRewards), difRewards.Text('f', precisionRewards) -} - -// CalculateRewardsPerHour will return an approximation of how many ERDs a validator will earn per hour -// Rewards estimation per hour will be equals with : -// chanceToBeInConsensus * roundsPerHour * hitRate * denominationCoefficient * Rewards -func (psh *PresenterStatusHandler) CalculateRewardsPerHour() string { - chanceToBeInConsensus := psh.computeChanceToBeInConsensus() - roundsPerHourAccordingToHitRate := psh.computeRoundsPerHourAccordingToHitRate() - rewardsInErd := psh.computeRewardsInErd() - if chanceToBeInConsensus == 0 || roundsPerHourAccordingToHitRate == 0 || rewardsInErd.Cmp(big.NewFloat(0)) <= 0 { - return "0" - } - - rewardsPerHourCoefficient := chanceToBeInConsensus * roundsPerHourAccordingToHitRate - totalRewardsPerHourFloat := big.NewFloat(rewardsPerHourCoefficient) - totalRewardsPerHourFloat.Mul(totalRewardsPerHourFloat, rewardsInErd) - return totalRewardsPerHourFloat.Text('f', precisionRewards) -} - -// GetZeros will return a string with a specific number of zeros -func (psh *PresenterStatusHandler) GetZeros() string { - retValue := "." + strings.Repeat("0", precisionRewards) - if retValue == "." { - return "" - } - - return retValue -} diff --git a/cmd/termui/presenter/instanceInfoGetters_test.go b/cmd/termui/presenter/instanceInfoGetters_test.go index 7681f21baf9..7912bc735dd 100644 --- a/cmd/termui/presenter/instanceInfoGetters_test.go +++ b/cmd/termui/presenter/instanceInfoGetters_test.go @@ -1,7 +1,6 @@ package presenter import ( - "math/big" "testing" "github.com/ElrondNetwork/elrond-go/common" @@ -149,73 +148,3 @@ func TestPresenterStatusHandler_GetNodeName(t *testing.T) { assert.Equal(t, nodeName, result) } - -func TestPresenterStatusHandler_CalculateRewardsTotal(t *testing.T) { - t.Parallel() - - rewardsValue := "1000" - - numSignedBlocks := uint64(50) - - presenterStatusHandler := NewPresenterStatusHandler() - presenterStatusHandler.SetStringValue(common.MetricRewardsValue, rewardsValue) - presenterStatusHandler.SetUInt64Value(common.MetricCountConsensusAcceptedBlocks, numSignedBlocks) - presenterStatusHandler.SetUInt64Value(common.MetricDenomination, 4) - totalRewards, diff := presenterStatusHandler.GetTotalRewardsValue() - expectedDifValue := "5" + presenterStatusHandler.GetZeros() - - assert.Equal(t, "0"+presenterStatusHandler.GetZeros(), totalRewards) - assert.Equal(t, expectedDifValue, diff) -} - -func TestPresenterStatusHandler_CalculateRewardsTotalRewards(t *testing.T) { - t.Parallel() - - rewardsValue := "1000" - numSignedBlocks := uint64(50000) - - presenterStatusHandler := NewPresenterStatusHandler() - totalRewardsOld, _ := big.NewFloat(0).SetString(rewardsValue) - presenterStatusHandler.totalRewardsOld = big.NewFloat(0).Set(totalRewardsOld) - presenterStatusHandler.SetStringValue(common.MetricRewardsValue, rewardsValue) - presenterStatusHandler.SetUInt64Value(common.MetricCountConsensusAcceptedBlocks, numSignedBlocks) - presenterStatusHandler.SetUInt64Value(common.MetricDenomination, 4) - totalRewards, diff := presenterStatusHandler.GetTotalRewardsValue() - expectedDiffValue := "4000" + presenterStatusHandler.GetZeros() - - assert.Equal(t, totalRewardsOld.Text('f', precisionRewards), totalRewards) - assert.Equal(t, expectedDiffValue, diff) -} - -func TestPresenterStatusHandler_CalculateRewardsPerHourReturnZero(t *testing.T) { - t.Parallel() - - presenterStatusHandler := NewPresenterStatusHandler() - result := presenterStatusHandler.CalculateRewardsPerHour() - - assert.Equal(t, "0", result) -} - -func TestPresenterStatusHandler_CalculateRewardsPerHourShouldWork(t *testing.T) { - t.Parallel() - - consensusGroupSize := uint64(50) - numValidators := uint64(100) - totalBlocks := uint64(1000) - totalRounds := uint64(1000) - roundTime := uint64(6) - rewardsValue := "10000" - - presenterStatusHandler := NewPresenterStatusHandler() - presenterStatusHandler.SetUInt64Value(common.MetricConsensusGroupSize, consensusGroupSize) - presenterStatusHandler.SetUInt64Value(common.MetricNumValidators, numValidators) - presenterStatusHandler.SetUInt64Value(common.MetricProbableHighestNonce, totalBlocks) - presenterStatusHandler.SetStringValue(common.MetricRewardsValue, rewardsValue) - presenterStatusHandler.SetUInt64Value(common.MetricCurrentRound, totalRounds) - presenterStatusHandler.SetUInt64Value(common.MetricRoundTime, roundTime) - presenterStatusHandler.SetUInt64Value(common.MetricDenomination, 4) - expectedValue := "300" + presenterStatusHandler.GetZeros() - - result := presenterStatusHandler.CalculateRewardsPerHour() - assert.Equal(t, expectedValue, result) -} diff --git a/cmd/termui/view/interface.go b/cmd/termui/view/interface.go index 3c331b74750..e2c33976593 100644 --- a/cmd/termui/view/interface.go +++ b/cmd/termui/view/interface.go @@ -57,10 +57,6 @@ type Presenter interface { GetNetworkSentBytesInEpoch() uint64 GetNetworkReceivedBytesInEpoch() uint64 - GetTotalRewardsValue() (string, string) - CalculateRewardsPerHour() string - GetZeros() string - InvalidateCache() IsInterfaceNil() bool } diff --git a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go index 171b82b5176..e54338e6ab6 100644 --- a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go +++ b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go @@ -166,30 +166,6 @@ func (wr *WidgetsRender) prepareInstanceInfo() { rows[4] = []string{fmt.Sprintf("Blocks proposed: %d | Blocks accepted: %d", countLeader, countAcceptedBlocks)} rows[5] = []string{computeRedundancyStr(wr.presenter.GetRedundancyLevel(), wr.presenter.GetRedundancyIsMainActive())} - - // TODO: repair the rewards estimation or replace these 2 rows with rating details - //switch instanceType { - //case string(common.NodeTypeValidator): - // rewardsPerHour := wr.presenter.CalculateRewardsPerHour() - // rows[5] = []string{fmt.Sprintf("Rewards estimation: %s ERD/h (without fees)", rewardsPerHour)} - // - // var rewardsInfo []string - // totalRewardsValue, diffRewards := wr.presenter.GetTotalRewardsValue() - // zeroString := "0" + wr.presenter.GetZeros() - // if diffRewards != zeroString { - // wr.instanceInfo.RowStyles[7] = ui.NewStyle(ui.ColorGreen) - // rewardsInfo = []string{fmt.Sprintf("Total rewards %s + %s ERD (without fees)", totalRewardsValue, diffRewards)} - // } else { - // wr.instanceInfo.RowStyles[7] = ui.NewStyle(ui.ColorWhite) - // rewardsInfo = []string{fmt.Sprintf("Total rewards %s ERD (without fees)", totalRewardsValue)} - // } - // rows[6] = rewardsInfo - // - //default: - // rows[5] = []string{""} - // rows[6] = []string{""} - //} - rows[6] = []string{""} wr.instanceInfo.Title = "Elrond instance info" @@ -215,7 +191,7 @@ func (wr *WidgetsRender) prepareChainInfo(numMillisecondsRefreshTime int) { blocksPerSecond := wr.presenter.CalculateSynchronizationSpeed(numMillisecondsRefreshTime) blocksPerSecondMessage = fmt.Sprintf("%d blocks/sec", blocksPerSecond) - case synchronizedRound == currentRound && currentRound == 0: + case currentRound == 0: syncingStr = statusNotApplicable default: syncingStr = statusSynchronized @@ -279,7 +255,7 @@ func computeRedundancyStr(redundancyLevel uint64, redundancyIsMainActive string) redundancyStr += "inactive" } else { if redundancyLevel == 0 { - redundancyStr += "Main machine" + redundancyStr += "main machine" } else { redundancyStr += fmt.Sprintf("back-up #%d", redundancyLevel) redundancyStr += fmt.Sprintf(" (is main active: %s)", redundancyIsMainActive) From c477ff754f2c44950edd86a829d5f1413f3a26e2 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 20 Apr 2022 09:36:31 +0300 Subject: [PATCH 13/19] indexer v1.1.41 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 849f427e28c..027f9d2c9ee 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ElrondNetwork/arwen-wasm-vm/v1_4 v1.4.34-rc7 github.com/ElrondNetwork/concurrent-map v0.1.3 github.com/ElrondNetwork/covalent-indexer-go v1.0.6 - github.com/ElrondNetwork/elastic-indexer-go v1.1.40 + github.com/ElrondNetwork/elastic-indexer-go v1.1.41-0.20220419195249-74671c9b0c6c github.com/ElrondNetwork/elrond-go-core v1.1.14 github.com/ElrondNetwork/elrond-go-crypto v1.0.1 github.com/ElrondNetwork/elrond-go-logger v1.0.5 diff --git a/go.sum b/go.sum index 2dbab9318d6..9562c3df8ee 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/ElrondNetwork/concurrent-map v0.1.3 h1:j2LtPrNJuerannC1cQDE79STvi/P04 github.com/ElrondNetwork/concurrent-map v0.1.3/go.mod h1:3XwSwn4JHI0lrKxWLZvtp53Emr8BXYTmNQGwcukHJEE= github.com/ElrondNetwork/covalent-indexer-go v1.0.6 h1:+LNKItUc+Pb7WuTbil3VuiLMmdQ1AY7lBJM476PtVNE= github.com/ElrondNetwork/covalent-indexer-go v1.0.6/go.mod h1:j3h2g96vqhJAuj3aEX2PWhomae2/o7YfXGEfweNXEeQ= -github.com/ElrondNetwork/elastic-indexer-go v1.1.40 h1:0M0G7Nct4vnuiIn8uN2efdIZmo1Hu3kQLWipSMcQTRU= -github.com/ElrondNetwork/elastic-indexer-go v1.1.40/go.mod h1:zLa7vRvTJXjGXZuOy0BId3v+fvn5LSibOC2BeTsCqvs= +github.com/ElrondNetwork/elastic-indexer-go v1.1.41-0.20220419195249-74671c9b0c6c h1:x4wx4HZFyLUiLM1LxdQvAug9i3YiGopLxD4jD9DoHGs= +github.com/ElrondNetwork/elastic-indexer-go v1.1.41-0.20220419195249-74671c9b0c6c/go.mod h1:zLa7vRvTJXjGXZuOy0BId3v+fvn5LSibOC2BeTsCqvs= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.6/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= From e8defdf398d988e36f25a66e749db9268ecf2326 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 20 Apr 2022 10:56:28 +0300 Subject: [PATCH 14/19] small fix --- process/smartContract/testScProcessor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/process/smartContract/testScProcessor.go b/process/smartContract/testScProcessor.go index 04eb5774787..244c0404741 100644 --- a/process/smartContract/testScProcessor.go +++ b/process/smartContract/testScProcessor.go @@ -29,7 +29,6 @@ func (tsp *TestScProcessor) GetLatestTestError() error { for _, logs := range allLogs { for _, event := range logs.GetLogEvents() { if string(event.GetIdentifier()) == signalError { - tsp.txLogsProcessor.Clean() return fmt.Errorf(string(event.GetTopics()[1])) } } From f49cd25f9a598a5ae4a8e08902f2df879f74734f Mon Sep 17 00:00:00 2001 From: bogdan-rosianu Date: Wed, 20 Apr 2022 11:43:35 +0300 Subject: [PATCH 15/19] redundancy level fixes --- cmd/termui/presenter/instanceInfoGetters.go | 12 ++++++++++-- .../presenter/instanceInfoGetters_test.go | 17 +++++++++++++++++ cmd/termui/view/interface.go | 2 +- .../view/termuic/termuiRenders/widgetsRender.go | 2 +- node/nodeRunner.go | 2 +- 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/cmd/termui/presenter/instanceInfoGetters.go b/cmd/termui/presenter/instanceInfoGetters.go index 37e80a24dba..e9bb807ba37 100644 --- a/cmd/termui/presenter/instanceInfoGetters.go +++ b/cmd/termui/presenter/instanceInfoGetters.go @@ -1,6 +1,7 @@ package presenter import ( + "strconv" "strings" "github.com/ElrondNetwork/elrond-go/common" @@ -32,8 +33,15 @@ func (psh *PresenterStatusHandler) GetPublicKeyBlockSign() string { } // GetRedundancyLevel will return the redundancy level of the node -func (psh *PresenterStatusHandler) GetRedundancyLevel() uint64 { - return psh.getFromCacheAsUint64(common.MetricRedundancyLevel) +func (psh *PresenterStatusHandler) GetRedundancyLevel() int64 { + // redundancy level is sent as string as JSON unmarshal doesn't treat well the casting from interface{} to int64 + redundancyLevelStr := psh.getFromCacheAsString(common.MetricRedundancyLevel) + i64Val, err := strconv.ParseInt(redundancyLevelStr, 10, 64) + if err != nil { + return 0 + } + + return i64Val } // GetRedundancyIsMainActive will return the info about redundancy main machine diff --git a/cmd/termui/presenter/instanceInfoGetters_test.go b/cmd/termui/presenter/instanceInfoGetters_test.go index 7912bc735dd..5f96feab8c3 100644 --- a/cmd/termui/presenter/instanceInfoGetters_test.go +++ b/cmd/termui/presenter/instanceInfoGetters_test.go @@ -5,6 +5,7 @@ import ( "github.com/ElrondNetwork/elrond-go/common" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPresenterStatusHandler_GetAppVersion(t *testing.T) { @@ -148,3 +149,19 @@ func TestPresenterStatusHandler_GetNodeName(t *testing.T) { assert.Equal(t, nodeName, result) } + +func TestPresenterStatusHandler_GetRedundancyLevel(t *testing.T) { + t.Parallel() + + testRedundancyParsing(t, "-1", -1) + testRedundancyParsing(t, "0", 0) + testRedundancyParsing(t, "invalid", 0) + testRedundancyParsing(t, "1", 1) +} + +func testRedundancyParsing(t *testing.T, input string, desiredOutput int64) { + psh := NewPresenterStatusHandler() + psh.SetStringValue(common.MetricRedundancyLevel, input) + redLev := psh.GetRedundancyLevel() + require.Equal(t, desiredOutput, redLev) +} diff --git a/cmd/termui/view/interface.go b/cmd/termui/view/interface.go index e2c33976593..006ac02c322 100644 --- a/cmd/termui/view/interface.go +++ b/cmd/termui/view/interface.go @@ -5,7 +5,7 @@ type Presenter interface { GetAppVersion() string GetNodeName() string GetPublicKeyBlockSign() string - GetRedundancyLevel() uint64 + GetRedundancyLevel() int64 GetRedundancyIsMainActive() string GetShardId() uint64 GetNodeType() string diff --git a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go index e54338e6ab6..0bcc478948e 100644 --- a/cmd/termui/view/termuic/termuiRenders/widgetsRender.go +++ b/cmd/termui/view/termuic/termuiRenders/widgetsRender.go @@ -245,7 +245,7 @@ func (wr *WidgetsRender) prepareChainInfo(numMillisecondsRefreshTime int) { wr.chainInfo.Rows = rows } -func computeRedundancyStr(redundancyLevel uint64, redundancyIsMainActive string) string { +func computeRedundancyStr(redundancyLevel int64, redundancyIsMainActive string) string { if redundancyIsMainActive == statusNotApplicable { return "" } diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 678b7261a33..410d3b305fb 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -587,7 +587,7 @@ func (nr *nodeRunner) createMetrics( } metrics.SaveStringMetric(coreComponents.StatusHandler(), common.MetricNodeDisplayName, nr.configs.PreferencesConfig.Preferences.NodeDisplayName) - metrics.SaveUint64Metric(coreComponents.StatusHandler(), common.MetricRedundancyLevel, uint64(nr.configs.PreferencesConfig.Preferences.RedundancyLevel)) + metrics.SaveStringMetric(coreComponents.StatusHandler(), common.MetricRedundancyLevel, fmt.Sprintf("%d", nr.configs.PreferencesConfig.Preferences.RedundancyLevel)) metrics.SaveStringMetric(coreComponents.StatusHandler(), common.MetricRedundancyIsMainActive, common.MetricValueNA) metrics.SaveStringMetric(coreComponents.StatusHandler(), common.MetricChainId, coreComponents.ChainID()) metrics.SaveUint64Metric(coreComponents.StatusHandler(), common.MetricGasPerDataByte, coreComponents.EconomicsData().GasPerDataByte()) From b4c68ee3d2ea223f7c0dba4205e8738bf2e641f4 Mon Sep 17 00:00:00 2001 From: Iuga Mihai Date: Wed, 20 Apr 2022 13:49:07 +0300 Subject: [PATCH 16/19] propper release --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 027f9d2c9ee..49e4e37f9a5 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ElrondNetwork/arwen-wasm-vm/v1_4 v1.4.34-rc7 github.com/ElrondNetwork/concurrent-map v0.1.3 github.com/ElrondNetwork/covalent-indexer-go v1.0.6 - github.com/ElrondNetwork/elastic-indexer-go v1.1.41-0.20220419195249-74671c9b0c6c + github.com/ElrondNetwork/elastic-indexer-go v1.1.41 github.com/ElrondNetwork/elrond-go-core v1.1.14 github.com/ElrondNetwork/elrond-go-crypto v1.0.1 github.com/ElrondNetwork/elrond-go-logger v1.0.5 diff --git a/go.sum b/go.sum index 9562c3df8ee..4647e95fd54 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/ElrondNetwork/concurrent-map v0.1.3 h1:j2LtPrNJuerannC1cQDE79STvi/P04 github.com/ElrondNetwork/concurrent-map v0.1.3/go.mod h1:3XwSwn4JHI0lrKxWLZvtp53Emr8BXYTmNQGwcukHJEE= github.com/ElrondNetwork/covalent-indexer-go v1.0.6 h1:+LNKItUc+Pb7WuTbil3VuiLMmdQ1AY7lBJM476PtVNE= github.com/ElrondNetwork/covalent-indexer-go v1.0.6/go.mod h1:j3h2g96vqhJAuj3aEX2PWhomae2/o7YfXGEfweNXEeQ= -github.com/ElrondNetwork/elastic-indexer-go v1.1.41-0.20220419195249-74671c9b0c6c h1:x4wx4HZFyLUiLM1LxdQvAug9i3YiGopLxD4jD9DoHGs= -github.com/ElrondNetwork/elastic-indexer-go v1.1.41-0.20220419195249-74671c9b0c6c/go.mod h1:zLa7vRvTJXjGXZuOy0BId3v+fvn5LSibOC2BeTsCqvs= +github.com/ElrondNetwork/elastic-indexer-go v1.1.41 h1:FDE+eIxc8zEwu1sZ9mgij2Ci3Fb2H2VV8EkfF9D5ZO4= +github.com/ElrondNetwork/elastic-indexer-go v1.1.41/go.mod h1:zLa7vRvTJXjGXZuOy0BId3v+fvn5LSibOC2BeTsCqvs= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.6/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= From 606f5f7cecdb4399a2dee55d58f519e7dd303fc4 Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Wed, 20 Apr 2022 21:43:53 +0300 Subject: [PATCH 17/19] - fixed integration tests by calling `tsp.txLogsProcessor.Clean()` in testScProcessor - changed the testScProcessor`s GetLatestTestError to return all errors it finds --- .../vm/arwen/arwenvm/arwenVM_test.go | 2 +- .../arwen/badcontracts/badcontracts_test.go | 1 + integrationTests/vm/arwen/utils.go | 12 ++++----- integrationTests/vm/testInitializer.go | 6 ++--- .../vm/txsFee/multiESDTTransfer_test.go | 6 ++--- .../vm/txsFee/multiShard/moveBalance_test.go | 4 +-- integrationTests/vm/txsFee/scDeploy_test.go | 4 +-- process/smartContract/testScProcessor.go | 26 +++++++++++++------ 8 files changed, 36 insertions(+), 25 deletions(-) diff --git a/integrationTests/vm/arwen/arwenvm/arwenVM_test.go b/integrationTests/vm/arwen/arwenvm/arwenVM_test.go index 57b025ea544..e07ce258bbb 100644 --- a/integrationTests/vm/arwen/arwenvm/arwenVM_test.go +++ b/integrationTests/vm/arwen/arwenvm/arwenVM_test.go @@ -182,7 +182,7 @@ func TestSCMoveBalanceBeforeSCDeploy(t *testing.T) { _, err = testContext.TxProcessor.ProcessTransaction(tx) require.Equal(t, process.ErrFailedTransaction, err) - require.Equal(t, process.ErrAccountNotPayable, testContext.GetLatestError()) + require.Equal(t, fmt.Errorf("%s: %s", process.ErrAccountNotPayable.Error(), "sending value to non payable contract"), testContext.GetCompositeTestError()) vm.TestAccount( t, testContext.Accounts, diff --git a/integrationTests/vm/arwen/badcontracts/badcontracts_test.go b/integrationTests/vm/arwen/badcontracts/badcontracts_test.go index 6b172fed92b..89e9efa8a81 100644 --- a/integrationTests/vm/arwen/badcontracts/badcontracts_test.go +++ b/integrationTests/vm/arwen/badcontracts/badcontracts_test.go @@ -1,3 +1,4 @@ +//go:build !race // +build !race package badcontracts diff --git a/integrationTests/vm/arwen/utils.go b/integrationTests/vm/arwen/utils.go index 72bbabffa8b..99881601888 100644 --- a/integrationTests/vm/arwen/utils.go +++ b/integrationTests/vm/arwen/utils.go @@ -496,7 +496,7 @@ func (context *TestContext) DeploySC(wasmPath string, parametersString string) e return err } - err = context.GetLatestError() + err = context.GetCompositeTestError() if err != nil { return err } @@ -550,7 +550,7 @@ func (context *TestContext) UpgradeSC(wasmPath string, parametersString string) return err } - err = context.GetLatestError() + err = context.GetCompositeTestError() if err != nil { return err } @@ -626,7 +626,7 @@ func (context *TestContext) ExecuteSCWithValue(sender *testParticipant, txData s return err } - err = context.GetLatestError() + err = context.GetCompositeTestError() if err != nil { return err } @@ -698,9 +698,9 @@ func (context *TestContext) GoToEpoch(epoch int) { context.BlockchainHook.SetCurrentHeader(header) } -// GetLatestError - -func (context *TestContext) GetLatestError() error { - return context.ScProcessor.GetLatestTestError() +// GetCompositeTestError - +func (context *TestContext) GetCompositeTestError() error { + return context.ScProcessor.GetCompositeTestError() } // FormatHexNumber - diff --git a/integrationTests/vm/testInitializer.go b/integrationTests/vm/testInitializer.go index 311ee1f39d0..f221cc3ee18 100644 --- a/integrationTests/vm/testInitializer.go +++ b/integrationTests/vm/testInitializer.go @@ -130,9 +130,9 @@ func (vmTestContext *VMTestContext) Close() { _ = vmTestContext.VMContainer.Close() } -// GetLatestError - -func (vmTestContext *VMTestContext) GetLatestError() error { - return vmTestContext.ScProcessor.GetLatestTestError() +// GetCompositeTestError - +func (vmTestContext *VMTestContext) GetCompositeTestError() error { + return vmTestContext.ScProcessor.GetCompositeTestError() } // CreateBlockStarted - diff --git a/integrationTests/vm/txsFee/multiESDTTransfer_test.go b/integrationTests/vm/txsFee/multiESDTTransfer_test.go index 8da52ea29fa..53dd7e2b014 100644 --- a/integrationTests/vm/txsFee/multiESDTTransfer_test.go +++ b/integrationTests/vm/txsFee/multiESDTTransfer_test.go @@ -39,7 +39,7 @@ func TestMultiESDTTransferShouldWork(t *testing.T) { retCode, err := testContext.TxProcessor.ProcessTransaction(tx) require.Equal(t, vmcommon.Ok, retCode) require.Nil(t, err) - require.Nil(t, testContext.GetLatestError()) + require.Nil(t, testContext.GetCompositeTestError()) _, err = testContext.Accounts.Commit() require.Nil(t, err) @@ -68,8 +68,8 @@ func TestMultiESDTTransferShouldWork(t *testing.T) { testIndexer.SaveTransaction(tx, block.TxBlock, intermediateTxs) indexerTx := testIndexer.GetIndexerPreparedTransaction(t) - require.Equal(t, uint64(4000), indexerTx.GasUsed) - require.Equal(t, "40000", indexerTx.Fee) + require.Equal(t, uint64(133), indexerTx.GasUsed) + require.Equal(t, "1330", indexerTx.Fee) allLogs := testContext.TxsLogsProcessor.GetAllCurrentLogs() require.NotNil(t, allLogs) diff --git a/integrationTests/vm/txsFee/multiShard/moveBalance_test.go b/integrationTests/vm/txsFee/multiShard/moveBalance_test.go index 25ab8183c40..f3993c1b231 100644 --- a/integrationTests/vm/txsFee/multiShard/moveBalance_test.go +++ b/integrationTests/vm/txsFee/multiShard/moveBalance_test.go @@ -84,7 +84,7 @@ func TestMoveBalanceContractAddressDataFieldNilShouldConsumeGas(t *testing.T) { retCode, err := testContext.TxProcessor.ProcessTransaction(tx) require.Equal(t, vmcommon.UserError, retCode) require.Nil(t, err) - require.Equal(t, errors.New("sending value to non payable contract"), testContext.GetLatestError()) + require.Equal(t, errors.New("sending value to non payable contract: sending value to non payable contract"), testContext.GetCompositeTestError()) _, err = testContext.Accounts.Commit() require.Nil(t, err) @@ -133,7 +133,7 @@ func TestMoveBalanceContractAddressDataFieldNotNilShouldConsumeGas(t *testing.T) retCode, err := testContext.TxProcessor.ProcessTransaction(tx) require.Equal(t, vmcommon.UserError, retCode) require.Nil(t, err) - require.Equal(t, errors.New("invalid contract code (not found)"), testContext.GetLatestError()) + require.Equal(t, errors.New("invalid contract code (not found): contract not found"), testContext.GetCompositeTestError()) _, err = testContext.Accounts.Commit() require.Nil(t, err) diff --git a/integrationTests/vm/txsFee/scDeploy_test.go b/integrationTests/vm/txsFee/scDeploy_test.go index a2efd975c7c..d85e8c11137 100644 --- a/integrationTests/vm/txsFee/scDeploy_test.go +++ b/integrationTests/vm/txsFee/scDeploy_test.go @@ -36,7 +36,7 @@ func TestScDeployShouldWork(t *testing.T) { _, err = testContext.TxProcessor.ProcessTransaction(tx) require.Nil(t, err) - require.Nil(t, testContext.GetLatestError()) + require.Nil(t, testContext.GetCompositeTestError()) _, err = testContext.Accounts.Commit() require.Nil(t, err) @@ -118,7 +118,7 @@ func TestScDeployInsufficientGasLimitShouldNotConsumeGas(t *testing.T) { _, err = testContext.TxProcessor.ProcessTransaction(tx) require.Equal(t, process.ErrInsufficientGasLimitInTx, err) - require.Nil(t, testContext.GetLatestError()) + require.Nil(t, testContext.GetCompositeTestError()) _, err = testContext.Accounts.Commit() require.Nil(t, err) diff --git a/process/smartContract/testScProcessor.go b/process/smartContract/testScProcessor.go index 244c0404741..7699ce421ec 100644 --- a/process/smartContract/testScProcessor.go +++ b/process/smartContract/testScProcessor.go @@ -21,15 +21,16 @@ func NewTestScProcessor(internalData *scProcessor) *TestScProcessor { return &TestScProcessor{internalData} } -// GetLatestTestError locates the latest error in the collection of smart contracts results -func (tsp *TestScProcessor) GetLatestTestError() error { +// GetCompositeTestError composes all errors found in the logs or by parsing the scr forwarder's contents +func (tsp *TestScProcessor) GetCompositeTestError() error { + var returnError error if tsp.flagCleanUpInformativeSCRs.IsSet() { allLogs := tsp.txLogsProcessor.GetAllCurrentLogs() for _, logs := range allLogs { for _, event := range logs.GetLogEvents() { if string(event.GetIdentifier()) == signalError { - return fmt.Errorf(string(event.GetTopics()[1])) + returnError = wrapError(returnError, string(event.GetTopics()[1])) } } } @@ -39,7 +40,7 @@ func (tsp *TestScProcessor) GetLatestTestError() error { GetIntermediateTransactions() []data.TransactionHandler }) if !ok { - return nil + return returnError } scResults := scrProvider.GetIntermediateTransactions() @@ -64,17 +65,26 @@ func (tsp *TestScProcessor) GetLatestTestError() error { if err == nil { returnCodeAsString := string(returnCode) if returnCodeAsString == "ok" || returnCodeAsString == "" { - return nil + return returnError } - return fmt.Errorf(returnCodeAsString) + return wrapError(returnError, returnCodeAsString) } - return fmt.Errorf(returnCodeHex) + return wrapError(returnError, returnCodeHex) } + tsp.txLogsProcessor.Clean() tsp.scrForwarder.CreateBlockStarted() - return nil + return returnError +} + +func wrapError(originalError error, msg string) error { + if originalError == nil { + return fmt.Errorf(msg) + } + + return fmt.Errorf("%s: %s", originalError.Error(), msg) } // GetGasRemaining returns the remaining gas from the last transaction From 97935e551768547b8534ab5efe17075e7da24dae Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Wed, 20 Apr 2022 22:19:11 +0300 Subject: [PATCH 18/19] - wrap error only if missing - changed github workflows scope --- .github/workflows/code-coverage.yml | 1 + .github/workflows/golangci-lint.yml | 1 + integrationTests/vm/arwen/arwenvm/arwenVM_test.go | 2 +- .../vm/txsFee/multiShard/moveBalance_test.go | 2 +- process/smartContract/testScProcessor.go | 13 +++++++++---- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index ed210c82495..d323b289653 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -6,6 +6,7 @@ on: - master pull_request: branches: [ master, development ] + types: [opened, ready_for_review] workflow_dispatch: jobs: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 962a0df83d4..0697352b1a0 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -5,6 +5,7 @@ on: - master pull_request: branches: [ master, development ] + types: [opened, ready_for_review] jobs: golangci: name: golangci linter diff --git a/integrationTests/vm/arwen/arwenvm/arwenVM_test.go b/integrationTests/vm/arwen/arwenvm/arwenVM_test.go index e07ce258bbb..d8285c1afb7 100644 --- a/integrationTests/vm/arwen/arwenvm/arwenVM_test.go +++ b/integrationTests/vm/arwen/arwenvm/arwenVM_test.go @@ -182,7 +182,7 @@ func TestSCMoveBalanceBeforeSCDeploy(t *testing.T) { _, err = testContext.TxProcessor.ProcessTransaction(tx) require.Equal(t, process.ErrFailedTransaction, err) - require.Equal(t, fmt.Errorf("%s: %s", process.ErrAccountNotPayable.Error(), "sending value to non payable contract"), testContext.GetCompositeTestError()) + require.Equal(t, process.ErrAccountNotPayable, testContext.GetCompositeTestError()) vm.TestAccount( t, testContext.Accounts, diff --git a/integrationTests/vm/txsFee/multiShard/moveBalance_test.go b/integrationTests/vm/txsFee/multiShard/moveBalance_test.go index f3993c1b231..cc153ab3407 100644 --- a/integrationTests/vm/txsFee/multiShard/moveBalance_test.go +++ b/integrationTests/vm/txsFee/multiShard/moveBalance_test.go @@ -84,7 +84,7 @@ func TestMoveBalanceContractAddressDataFieldNilShouldConsumeGas(t *testing.T) { retCode, err := testContext.TxProcessor.ProcessTransaction(tx) require.Equal(t, vmcommon.UserError, retCode) require.Nil(t, err) - require.Equal(t, errors.New("sending value to non payable contract: sending value to non payable contract"), testContext.GetCompositeTestError()) + require.Equal(t, errors.New("sending value to non payable contract"), testContext.GetCompositeTestError()) _, err = testContext.Accounts.Commit() require.Nil(t, err) diff --git a/process/smartContract/testScProcessor.go b/process/smartContract/testScProcessor.go index 7699ce421ec..e2f5b3de4e2 100644 --- a/process/smartContract/testScProcessor.go +++ b/process/smartContract/testScProcessor.go @@ -30,7 +30,7 @@ func (tsp *TestScProcessor) GetCompositeTestError() error { for _, logs := range allLogs { for _, event := range logs.GetLogEvents() { if string(event.GetIdentifier()) == signalError { - returnError = wrapError(returnError, string(event.GetTopics()[1])) + returnError = wrapErrorIfNotContains(returnError, string(event.GetTopics()[1])) } } } @@ -67,10 +67,10 @@ func (tsp *TestScProcessor) GetCompositeTestError() error { if returnCodeAsString == "ok" || returnCodeAsString == "" { return returnError } - return wrapError(returnError, returnCodeAsString) + return wrapErrorIfNotContains(returnError, returnCodeAsString) } - return wrapError(returnError, returnCodeHex) + return wrapErrorIfNotContains(returnError, returnCodeHex) } tsp.txLogsProcessor.Clean() @@ -79,11 +79,16 @@ func (tsp *TestScProcessor) GetCompositeTestError() error { return returnError } -func wrapError(originalError error, msg string) error { +func wrapErrorIfNotContains(originalError error, msg string) error { if originalError == nil { return fmt.Errorf(msg) } + alreadyContainsMessage := strings.Contains(originalError.Error(), msg) + if alreadyContainsMessage { + return originalError + } + return fmt.Errorf("%s: %s", originalError.Error(), msg) } From c20cc76e5b8d2b6ce6b4492f5b8a236bf05fb2bd Mon Sep 17 00:00:00 2001 From: Iulian Pascalau Date: Wed, 20 Apr 2022 22:24:16 +0300 Subject: [PATCH 19/19] - changed github workflows scope --- .github/workflows/code-coverage.yml | 4 ++-- .github/workflows/golangci-lint.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index d323b289653..db12153a29b 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -5,8 +5,8 @@ on: branches: - master pull_request: - branches: [ master, development ] - types: [opened, ready_for_review] + branches: [ master, development, feat/*, rc/* ] + workflow_dispatch: jobs: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 0697352b1a0..826346b5ec6 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -4,8 +4,8 @@ on: branches: - master pull_request: - branches: [ master, development ] - types: [opened, ready_for_review] + branches: [ master, development, feat/*, rc/* ] + jobs: golangci: name: golangci linter