Merge pull request #3986 from ElrondNetwork/feat-april-with-indexer-v…

…1-2-x

Indexer v1.2.25

cmd/node/config/external.toml

@@ -6,14 +6,14 @@
     #Strongly suggested to activate this on a regular observer node.
     Enabled           = false
     IndexerCacheSize  = 0
+    BulkRequestMaxSizeInBytes = 4194304 # 4MB
     URL               = "http://localhost:9200"
     UseKibana         = false
     Username          = ""
     Password          = ""
     # EnabledIndexes represents a slice of indexes that will be enabled for indexing. Full list is:
-    # ["tps", "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators"]
-    EnabledIndexes    = ["tps", "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators"]
-
+    # ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations"]
+    EnabledIndexes    = ["rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", "logs", "delegators", "operations"]
 # EventNotifierConnector defines settings needed to configure and launch the event notifier component
 [EventNotifierConnector]
     # Enabled will turn on or off the event notifier connector

config/externalConfig.go

@@ -9,13 +9,14 @@ type ExternalConfig struct {
 
 // ElasticSearchConfig will hold the configuration for the elastic search
 type ElasticSearchConfig struct {
-	Enabled          bool
-	IndexerCacheSize int
-	URL              string
-	UseKibana        bool
-	Username         string
-	Password         string
-	EnabledIndexes   []string
+	Enabled                   bool
+	IndexerCacheSize          int
+	BulkRequestMaxSizeInBytes int
+	URL                       string
+	UseKibana                 bool
+	Username                  string
+	Password                  string
+	EnabledIndexes            []string
 }
 
 // EventNotifierConfig will hold the configuration for the events notifier driver

factory/statusComponents.go

@@ -222,6 +222,7 @@ func (scf *statusComponentsFactory) makeElasticIndexerArgs() *indexerFactory.Arg
 	return &indexerFactory.ArgsIndexerFactory{
 		Enabled:                  elasticSearchConfig.Enabled,
 		IndexerCacheSize:         elasticSearchConfig.IndexerCacheSize,
+		BulkRequestMaxSize:       elasticSearchConfig.BulkRequestMaxSizeInBytes,
 		ShardCoordinator:         scf.shardCoordinator,
 		Url:                      elasticSearchConfig.URL,
 		UserName:                 elasticSearchConfig.Username,

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ElrondNetwork/arwen-wasm-vm/v1_4 v1.4.34-rc9
 	github.com/ElrondNetwork/concurrent-map v0.1.3
 	github.com/ElrondNetwork/covalent-indexer-go v1.0.6
-	github.com/ElrondNetwork/elastic-indexer-go v1.1.41
+	github.com/ElrondNetwork/elastic-indexer-go v1.2.25
 	github.com/ElrondNetwork/elrond-go-core v1.1.15
 	github.com/ElrondNetwork/elrond-go-crypto v1.0.1
 	github.com/ElrondNetwork/elrond-go-logger v1.0.7

go.sum

@@ -24,8 +24,8 @@ github.com/ElrondNetwork/concurrent-map v0.1.3 h1:j2LtPrNJuerannC1cQDE79STvi/P04
 github.com/ElrondNetwork/concurrent-map v0.1.3/go.mod h1:3XwSwn4JHI0lrKxWLZvtp53Emr8BXYTmNQGwcukHJEE=
 github.com/ElrondNetwork/covalent-indexer-go v1.0.6 h1:+LNKItUc+Pb7WuTbil3VuiLMmdQ1AY7lBJM476PtVNE=
 github.com/ElrondNetwork/covalent-indexer-go v1.0.6/go.mod h1:j3h2g96vqhJAuj3aEX2PWhomae2/o7YfXGEfweNXEeQ=
-github.com/ElrondNetwork/elastic-indexer-go v1.1.41 h1:FDE+eIxc8zEwu1sZ9mgij2Ci3Fb2H2VV8EkfF9D5ZO4=
-github.com/ElrondNetwork/elastic-indexer-go v1.1.41/go.mod h1:zLa7vRvTJXjGXZuOy0BId3v+fvn5LSibOC2BeTsCqvs=
+github.com/ElrondNetwork/elastic-indexer-go v1.2.25 h1:21ala1EQTu/30umkJxLTgIWikA17Iw8bP61EqK4poMo=
+github.com/ElrondNetwork/elastic-indexer-go v1.2.25/go.mod h1:XkrkGcomheEZyMC1/OoANQ9KV0OCZF6+UP8lSPRrE9I=
 github.com/ElrondNetwork/elrond-go-core v1.0.0/go.mod h1:FQMem7fFF4+8pQ6lVsBZq6yO+smD0nV23P4bJpmPjTo=
 github.com/ElrondNetwork/elrond-go-core v1.1.6/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g=
 github.com/ElrondNetwork/elrond-go-core v1.1.7/go.mod h1:O9FkkTT2H9kxCzfn40TbhoCDXzGmUrRVusMomhK/Y3g=
@@ -1010,6 +1010,12 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
+github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
+github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M=
 github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
 github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc=

integrationTests/mock/databaseWritterMock.go

@@ -14,6 +14,16 @@ type DatabaseWriterStub struct {
 	DoMultiGetCalled    func(ids []string, index string, withSource bool, res interface{}) error
 }
 
+// DoScrollRequest -
+func (dws *DatabaseWriterStub) DoScrollRequest(_ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error {
+	return nil
+}
+
+// DoCountRequest -
+func (dws *DatabaseWriterStub) DoCountRequest(_ string, _ []byte) (uint64, error) {
+	return 0, nil
+}
+
 // DoRequest -
 func (dws *DatabaseWriterStub) DoRequest(req *esapi.IndexRequest) error {
 	if dws.DoRequestCalled != nil {

integrationTests/vm/testIndexer.go

@@ -16,6 +16,7 @@ import (
 	blockProc "github.com/ElrondNetwork/elastic-indexer-go/process/block"
 	"github.com/ElrondNetwork/elastic-indexer-go/process/logsevents"
 	"github.com/ElrondNetwork/elastic-indexer-go/process/miniblocks"
+	"github.com/ElrondNetwork/elastic-indexer-go/process/operations"
 	"github.com/ElrondNetwork/elastic-indexer-go/process/statistics"
 	"github.com/ElrondNetwork/elastic-indexer-go/process/transactions"
 	"github.com/ElrondNetwork/elastic-indexer-go/process/validators"
@@ -124,7 +125,8 @@ func (ti *testIndexer) createElasticProcessor(
 	bp, _ := blockProc.NewBlockProcessor(testHasher, testMarshalizer)
 	mp, _ := miniblocks.NewMiniblocksProcessor(shardCoordinator.SelfId(), testHasher, testMarshalizer, false)
 	sp := statistics.NewStatisticsProcessor()
-	vp, _ := validators.NewValidatorsProcessor(pubkeyConv)
+	vp, _ := validators.NewValidatorsProcessor(pubkeyConv, 0)
+	opp, _ := operations.NewOperationsProcessor(false, shardCoordinator)
 	args := &logsevents.ArgsLogsAndEventsProcessor{
 		ShardCoordinator: shardCoordinator,
 		PubKeyConverter:  pubkeyConv,
@@ -149,6 +151,7 @@ func (ti *testIndexer) createElasticProcessor(
 		ValidatorsProc:    vp,
 		LogsAndEventsProc: lp,
 		DBClient:          databaseClient,
+		OperationsProc:    opp,
 	}
 
 	esProcessor, _ := elasticProcessor.NewElasticProcessor(esIndexerArgs)
@@ -254,6 +257,7 @@ func (ti *testIndexer) createDatabaseClient(hasResults bool) elasticProcessor.Da
 		ti.indexerData[index] = buff
 		if !done {
 			done = true
+			ti.saveDoneChan <- struct{}{}
 			return nil
 		}
 		ti.saveDoneChan <- struct{}{}
@@ -270,7 +274,7 @@ func (ti *testIndexer) createDatabaseClient(hasResults bool) elasticProcessor.Da
 // GetIndexerPreparedTransaction -
 func (ti *testIndexer) GetIndexerPreparedTransaction(t *testing.T) *indexerTypes.Transaction {
 	ti.mutex.RLock()
-	txData, ok := ti.indexerData["transactions"]
+	txData, ok := ti.indexerData[""]
 	ti.mutex.RUnlock()
 
 	require.True(t, ok)
@@ -300,7 +304,7 @@ func (ti *testIndexer) GetIndexerPreparedTransaction(t *testing.T) *indexerTypes
 
 func (ti *testIndexer) printReceipt() {
 	ti.mutex.RLock()
-	receipts, ok := ti.indexerData["receipts"]
+	receipts, ok := ti.indexerData[""]
 	ti.mutex.RUnlock()
 
 	if !ok {
@@ -319,7 +323,7 @@ func (ti *testIndexer) printReceipt() {
 
 func (ti *testIndexer) putSCRSInTx(tx *indexerTypes.Transaction) {
 	ti.mutex.RLock()
-	scrData, ok := ti.indexerData["scresults"]
+	scrData, ok := ti.indexerData[""]
 	ti.mutex.RUnlock()
 
 	if !ok {
@@ -330,6 +334,10 @@ func (ti *testIndexer) putSCRSInTx(tx *indexerTypes.Transaction) {
 	require.True(ti.t, len(split) > 2)
 
 	for idx := 1; idx < len(split); idx += 2 {
+		if !bytes.Contains(split[idx], []byte("scresults")) {
+			continue
+		}
+
 		newSCR := &indexerTypes.ScResult{}
 		err := json.Unmarshal(split[idx], newSCR)
 		require.Nil(ti.t, err)