diff --git a/cmd/node/config/external.toml b/cmd/node/config/external.toml index a7b4253660d..7902306a5dd 100644 --- a/cmd/node/config/external.toml +++ b/cmd/node/config/external.toml @@ -6,14 +6,14 @@ #Strongly suggested to activate this on a regular observer node. Enabled = false IndexerCacheSize = 0 + BulkRequestMaxSizeInBytes = 4194304 # 4MB URL = "http://localhost:9200" UseKibana = false Username = "" Password = "" # EnabledIndexes represents a slice of indexes that will be enabled for indexing. Full list is: - # ["tps", "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators"] - EnabledIndexes = ["tps", "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators"] - + # ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations"] + EnabledIndexes = ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations"] # EventNotifierConnector defines settings needed to configure and launch the event notifier component [EventNotifierConnector] # Enabled will turn on or off the event notifier connector diff --git a/config/externalConfig.go b/config/externalConfig.go index c1b4ca7857b..d4a869bdf4c 100644 --- a/config/externalConfig.go +++ b/config/externalConfig.go @@ -9,13 +9,14 @@ type ExternalConfig struct { // ElasticSearchConfig will hold the configuration for the elastic search type ElasticSearchConfig struct { - Enabled bool - IndexerCacheSize int - URL string - UseKibana bool - Username string - Password string - EnabledIndexes []string + Enabled bool + IndexerCacheSize int + BulkRequestMaxSizeInBytes int + URL string + UseKibana bool + Username string + Password string + EnabledIndexes []string } // EventNotifierConfig will hold the configuration for the events notifier driver diff --git a/factory/statusComponents.go b/factory/statusComponents.go index c5505a2bca7..0aa18d70a3d 100644 --- a/factory/statusComponents.go +++ b/factory/statusComponents.go @@ -222,6 +222,7 @@ func (scf *statusComponentsFactory) makeElasticIndexerArgs() *indexerFactory.Arg return &indexerFactory.ArgsIndexerFactory{ Enabled: elasticSearchConfig.Enabled, IndexerCacheSize: elasticSearchConfig.IndexerCacheSize, + BulkRequestMaxSize: elasticSearchConfig.BulkRequestMaxSizeInBytes, ShardCoordinator: scf.shardCoordinator, Url: elasticSearchConfig.URL, UserName: elasticSearchConfig.Username, diff --git a/go.mod b/go.mod index 2379baeec84..4047a64b30a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ElrondNetwork/arwen-wasm-vm/v1_4 v1.4.34-rc9 github.com/ElrondNetwork/concurrent-map v0.1.3 github.com/ElrondNetwork/covalent-indexer-go v1.0.6 - github.com/ElrondNetwork/elastic-indexer-go v1.1.41 + github.com/ElrondNetwork/elastic-indexer-go v1.2.25 github.com/ElrondNetwork/elrond-go-core v1.1.15 github.com/ElrondNetwork/elrond-go-crypto v1.0.1 github.com/ElrondNetwork/elrond-go-logger v1.0.7 diff --git a/go.sum b/go.sum index 4a225ea3165..a77de3020ff 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/ElrondNetwork/concurrent-map v0.1.3 h1:j2LtPrNJuerannC1cQDE79STvi/P04 github.com/ElrondNetwork/concurrent-map v0.1.3/go.mod h1:3XwSwn4JHI0lrKxWLZvtp53Emr8BXYTmNQGwcukHJEE= github.com/ElrondNetwork/covalent-indexer-go v1.0.6 h1:+LNKItUc+Pb7WuTbil3VuiLMmdQ1AY7lBJM476PtVNE= github.com/ElrondNetwork/covalent-indexer-go v1.0.6/go.mod h1:j3h2g96vqhJAuj3aEX2PWhomae2/o7YfXGEfweNXEeQ= -github.com/ElrondNetwork/elastic-indexer-go v1.1.41 h1:FDE+eIxc8zEwu1sZ9mgij2Ci3Fb2H2VV8EkfF9D5ZO4= -github.com/ElrondNetwork/elastic-indexer-go v1.1.41/go.mod h1:zLa7vRvTJXjGXZuOy0BId3v+fvn5LSibOC2BeTsCqvs= +github.com/ElrondNetwork/elastic-indexer-go v1.2.25 h1:21ala1EQTu/30umkJxLTgIWikA17Iw8bP61EqK4poMo= +github.com/ElrondNetwork/elastic-indexer-go v1.2.25/go.mod h1:XkrkGcomheEZyMC1/OoANQ9KV0OCZF6+UP8lSPRrE9I= github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo= github.com/ElrondNetwork/elrond-go-core v1.1.6/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g= @@ -1010,6 +1010,12 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= +github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= diff --git a/integrationTests/mock/databaseWritterMock.go b/integrationTests/mock/databaseWritterMock.go index 7dd22b555c2..d040b418fe2 100644 --- a/integrationTests/mock/databaseWritterMock.go +++ b/integrationTests/mock/databaseWritterMock.go @@ -14,6 +14,16 @@ type DatabaseWriterStub struct { DoMultiGetCalled func(ids []string, index string, withSource bool, res interface{}) error } +// DoScrollRequest - +func (dws *DatabaseWriterStub) DoScrollRequest(_ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error { + return nil +} + +// DoCountRequest - +func (dws *DatabaseWriterStub) DoCountRequest(_ string, _ []byte) (uint64, error) { + return 0, nil +} + // DoRequest - func (dws *DatabaseWriterStub) DoRequest(req *esapi.IndexRequest) error { if dws.DoRequestCalled != nil { diff --git a/integrationTests/vm/testIndexer.go b/integrationTests/vm/testIndexer.go index 92e0970e750..aa450f76dca 100644 --- a/integrationTests/vm/testIndexer.go +++ b/integrationTests/vm/testIndexer.go @@ -16,6 +16,7 @@ import ( blockProc "github.com/ElrondNetwork/elastic-indexer-go/process/block" "github.com/ElrondNetwork/elastic-indexer-go/process/logsevents" "github.com/ElrondNetwork/elastic-indexer-go/process/miniblocks" + "github.com/ElrondNetwork/elastic-indexer-go/process/operations" "github.com/ElrondNetwork/elastic-indexer-go/process/statistics" "github.com/ElrondNetwork/elastic-indexer-go/process/transactions" "github.com/ElrondNetwork/elastic-indexer-go/process/validators" @@ -124,7 +125,8 @@ func (ti *testIndexer) createElasticProcessor( bp, _ := blockProc.NewBlockProcessor(testHasher, testMarshalizer) mp, _ := miniblocks.NewMiniblocksProcessor(shardCoordinator.SelfId(), testHasher, testMarshalizer, false) sp := statistics.NewStatisticsProcessor() - vp, _ := validators.NewValidatorsProcessor(pubkeyConv) + vp, _ := validators.NewValidatorsProcessor(pubkeyConv, 0) + opp, _ := operations.NewOperationsProcessor(false, shardCoordinator) args := &logsevents.ArgsLogsAndEventsProcessor{ ShardCoordinator: shardCoordinator, PubKeyConverter: pubkeyConv, @@ -149,6 +151,7 @@ func (ti *testIndexer) createElasticProcessor( ValidatorsProc: vp, LogsAndEventsProc: lp, DBClient: databaseClient, + OperationsProc: opp, } esProcessor, _ := elasticProcessor.NewElasticProcessor(esIndexerArgs) @@ -254,6 +257,7 @@ func (ti *testIndexer) createDatabaseClient(hasResults bool) elasticProcessor.Da ti.indexerData[index] = buff if !done { done = true + ti.saveDoneChan <- struct{}{} return nil } ti.saveDoneChan <- struct{}{} @@ -270,7 +274,7 @@ func (ti *testIndexer) createDatabaseClient(hasResults bool) elasticProcessor.Da // GetIndexerPreparedTransaction - func (ti *testIndexer) GetIndexerPreparedTransaction(t *testing.T) *indexerTypes.Transaction { ti.mutex.RLock() - txData, ok := ti.indexerData["transactions"] + txData, ok := ti.indexerData[""] ti.mutex.RUnlock() require.True(t, ok) @@ -300,7 +304,7 @@ func (ti *testIndexer) GetIndexerPreparedTransaction(t *testing.T) *indexerTypes func (ti *testIndexer) printReceipt() { ti.mutex.RLock() - receipts, ok := ti.indexerData["receipts"] + receipts, ok := ti.indexerData[""] ti.mutex.RUnlock() if !ok { @@ -319,7 +323,7 @@ func (ti *testIndexer) printReceipt() { func (ti *testIndexer) putSCRSInTx(tx *indexerTypes.Transaction) { ti.mutex.RLock() - scrData, ok := ti.indexerData["scresults"] + scrData, ok := ti.indexerData[""] ti.mutex.RUnlock() if !ok { @@ -330,6 +334,10 @@ func (ti *testIndexer) putSCRSInTx(tx *indexerTypes.Transaction) { require.True(ti.t, len(split) > 2) for idx := 1; idx < len(split); idx += 2 { + if !bytes.Contains(split[idx], []byte("scresults")) { + continue + } + newSCR := &indexerTypes.ScResult{} err := json.Unmarshal(split[idx], newSCR) require.Nil(ti.t, err)