Merge pull request #256 from multiversx/rc/v1.7.0

`rc/v1.7.0`

cmd/elasticindexer/config/config.toml

@@ -2,7 +2,7 @@
     available-indices =  [
         "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory",
         "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags",
-        "logs", "delegators", "operations", "esdts"
+        "logs", "delegators", "operations", "esdts", "values", "events"
     ]
     [config.address-converter]
         length = 32

data/event.go

@@ -0,0 +1,20 @@
+package data
+
+import "time"
+
+// LogEvent is the dto for the log event structure
+type LogEvent struct {
+	ID             string        `json:"-"`
+	TxHash         string        `json:"txHash"`
+	OriginalTxHash string        `json:"originalTxHash,omitempty"`
+	LogAddress     string        `json:"logAddress"`
+	Address        string        `json:"address"`
+	Identifier     string        `json:"identifier"`
+	Data           string        `json:"data,omitempty"`
+	AdditionalData []string      `json:"additionalData,omitempty"`
+	Topics         []string      `json:"topics"`
+	Order          int           `json:"order"`
+	TxOrder        int           `json:"txOrder"`
+	ShardID        uint32        `json:"shardID"`
+	Timestamp      time.Duration `json:"timestamp,omitempty"`
+}

data/logs.go

@@ -38,4 +38,5 @@ type PreparedLogsResults struct {
 	NFTsDataUpdates         []*NFTDataUpdate
 	TokenRolesAndProperties *tokeninfo.TokenRolesAndProperties
 	DBLogs                  []*Logs
+	DBEvents                []*LogEvent
 }

data/scresult.go

@@ -39,6 +39,7 @@ type ScResult struct {
 	CanBeIgnored       bool          `json:"canBeIgnored,omitempty"`
 	OriginalSender     string        `json:"originalSender,omitempty"`
 	HasLogs            bool          `json:"hasLogs,omitempty"`
+	ExecutionOrder     int           `json:"-"`
 	SenderAddressBytes []byte        `json:"-"`
 	InitialTxGasUsed   uint64        `json:"-"`
 	InitialTxFee       string        `json:"-"`

data/transaction.go

@@ -5,8 +5,8 @@ import (
 )
 
 // Transaction is a structure containing all the fields that need
-//  to be saved for a transaction. It has all the default fields
-//  plus some extra information for ease of search and filter
+// to be saved for a transaction. It has all the default fields
+// plus some extra information for ease of search and filter
 type Transaction struct {
 	MBHash               string        `json:"miniBlockHash"`
 	Nonce                uint64        `json:"nonce"`
@@ -48,6 +48,7 @@ type Transaction struct {
 	GuardianSignature    string        `json:"guardianSignature,omitempty"`
 	ErrorEvent           bool          `json:"errorEvent,omitempty"`
 	CompletedEvent       bool          `json:"completedEvent,omitempty"`
+	ExecutionOrder       int           `json:"-"`
 	SmartContractResults []*ScResult   `json:"-"`
 	Hash                 string        `json:"-"`
 	BlockHash            string        `json:"-"`

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.0"
 services:
   elasticsearch:
     container_name: es-container

go.mod

@@ -6,10 +6,10 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.12.0
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-gonic/gin v1.9.1
-	github.com/multiversx/mx-chain-communication-go v1.0.12
-	github.com/multiversx/mx-chain-core-go v1.2.16
-	github.com/multiversx/mx-chain-logger-go v1.0.13
-	github.com/multiversx/mx-chain-vm-common-go v1.5.2
+	github.com/multiversx/mx-chain-communication-go v1.0.14
+	github.com/multiversx/mx-chain-core-go v1.2.19
+	github.com/multiversx/mx-chain-logger-go v1.0.14
+	github.com/multiversx/mx-chain-vm-common-go v1.5.12
 	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.37.0
 	github.com/stretchr/testify v1.8.4

go.sum

@@ -247,15 +247,15 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
-github.com/multiversx/mx-chain-communication-go v1.0.12 h1:67WOaf87gpwouydD1AAOHw5LMGZh7NfITrp/KqFY3Tw=
-github.com/multiversx/mx-chain-communication-go v1.0.12/go.mod h1:+oaUowpq+SqrEmAsMPGwhz44g7L81loWb6AiNQU9Ms4=
-github.com/multiversx/mx-chain-core-go v1.2.16 h1:m0hUNmZQjGJxKDLQOHoM9jSaeDfVTbyd+mqiS8+NckE=
-github.com/multiversx/mx-chain-core-go v1.2.16/go.mod h1:BILOGHUOIG5dNNX8cgkzCNfDaVtoYrJRYcPnpxRMH84=
-github.com/multiversx/mx-chain-crypto-go v1.2.8 h1:wOgVlUaO5X4L8iEbFjcQcL8SZvv6WZ7LqH73BiRPhxU=
-github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY=
-github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk=
-github.com/multiversx/mx-chain-vm-common-go v1.5.2 h1:iRWJNlogjkq9w+pJZIfkVkXQFmMoRxZr6pzCfg2/K68=
-github.com/multiversx/mx-chain-vm-common-go v1.5.2/go.mod h1:sqkKMCnwkWl8DURdb9q7pctK8IANghdHY1KJLE0ox2c=
+github.com/multiversx/mx-chain-communication-go v1.0.14 h1:YhAUDjBBpc5h5W0A7LHLXUMIMeCgwgGvkqfAPbFqsno=
+github.com/multiversx/mx-chain-communication-go v1.0.14/go.mod h1:qYCqgk0h+YpcTA84jHIpCBy6UShRwmXzHSCcdfwNrkw=
+github.com/multiversx/mx-chain-core-go v1.2.19 h1:2BaVHkB0tro3cjs5ay2pmLup1loCV0e1p9jV5QW0xqc=
+github.com/multiversx/mx-chain-core-go v1.2.19/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE=
+github.com/multiversx/mx-chain-crypto-go v1.2.11 h1:MNPJoiTJA5/tedYrI0N22OorbsKDESWG0SF8MCJwcJI=
+github.com/multiversx/mx-chain-logger-go v1.0.14 h1:PRMpAvXE7Nec2d//QNmbYfKVHMomOKmcN4UXurQWX9o=
+github.com/multiversx/mx-chain-logger-go v1.0.14/go.mod h1:bDfHSdwqIimn7Gp8w+SH5KlDuGzJ//nlyEANAaTSc3o=
+github.com/multiversx/mx-chain-vm-common-go v1.5.12 h1:Q8F6DE7XhgHtWgg2rozSv4Tv5fE3ENkJz6mjRoAfht8=
+github.com/multiversx/mx-chain-vm-common-go v1.5.12/go.mod h1:Sv6iS1okB6gy3HAsW6KHYtAxShNAfepKLtu//AURI8c=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=

integrationtests/accountsBalanceNftTransfer_test.go

@@ -34,6 +34,7 @@ func createOutportBlockWithHeader(
 			TransactionPool: pool,
 			AlteredAccounts: coreAlteredAccounts,
 			NumberOfShards:  numOfShards,
+			ShardID:         header.GetShardID(),
 		},
 		Header: header,
 	}

integrationtests/logsCrossShard_test.go

@@ -30,13 +30,21 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) {
 		Round:     50,
 		TimeStamp: 5040,
 	}
-	body := &dataBlock.Body{}
+
+	txHash := []byte("cross-log")
+	logID := hex.EncodeToString(txHash)
+
+	body := &dataBlock.Body{
+		MiniBlocks: []*dataBlock.MiniBlock{
+			{
+				TxHashes: [][]byte{txHash},
+			},
+		},
+	}
 
 	address1 := "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99"
 	address2 := "erd1w7jyzuj6cv4ngw8luhlkakatjpmjh3ql95lmxphd3vssc4vpymks6k5th7"
 
-	logID := hex.EncodeToString([]byte("cross-log"))
-
 	// index on source
 	pool := &outport.TransactionPool{
 		Logs: []*outport.LogData{
@@ -55,6 +63,12 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) {
 				},
 			},
 		},
+		Transactions: map[string]*outport.TxInfo{
+			logID: {
+				Transaction:    &transaction.Transaction{},
+				ExecutionOrder: 0,
+			},
+		},
 	}
 	err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards))
 	require.Nil(t, err)
@@ -68,10 +82,20 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) {
 		string(genericResponse.Docs[0].Source),
 	)
 
+	event1ID := logID + "-0-0"
+	ids = []string{event1ID}
+	err = esClient.DoMultiGet(context.Background(), ids, indexerdata.EventsIndex, true, genericResponse)
+	require.Nil(t, err)
+	require.JSONEq(t,
+		readExpectedResult("./testdata/logsCrossShard/event-transfer-source-first.json"),
+		string(genericResponse.Docs[0].Source),
+	)
+
 	// INDEX ON DESTINATION
 	header = &dataBlock.Header{
 		Round:     50,
 		TimeStamp: 6040,
+		ShardID:   1,
 	}
 	pool = &outport.TransactionPool{
 		Logs: []*outport.LogData{
@@ -96,17 +120,37 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) {
 				},
 			},
 		},
+		Transactions: map[string]*outport.TxInfo{
+			logID: {
+				Transaction:    &transaction.Transaction{},
+				ExecutionOrder: 0,
+			},
+		},
 	}
 	err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards))
 	require.Nil(t, err)
 
+	ids = []string{logID}
 	err = esClient.DoMultiGet(context.Background(), ids, indexerdata.LogsIndex, true, genericResponse)
 	require.Nil(t, err)
 	require.JSONEq(t,
 		readExpectedResult("./testdata/logsCrossShard/log-at-destination.json"),
 		string(genericResponse.Docs[0].Source),
 	)
 
+	event2ID, event3ID := logID+"-1-0", logID+"-1-1"
+	ids = []string{event2ID, event3ID}
+	err = esClient.DoMultiGet(context.Background(), ids, indexerdata.EventsIndex, true, genericResponse)
+	require.Nil(t, err)
+	require.JSONEq(t,
+		readExpectedResult("./testdata/logsCrossShard/event-transfer-destination.json"),
+		string(genericResponse.Docs[0].Source),
+	)
+	require.JSONEq(t,
+		readExpectedResult("./testdata/logsCrossShard/event-do-something.json"),
+		string(genericResponse.Docs[1].Source),
+	)
+
 	// index on source again should not change the log
 	header = &dataBlock.Header{
 		Round:     50,
@@ -129,10 +173,17 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) {
 				},
 			},
 		},
+		Transactions: map[string]*outport.TxInfo{
+			logID: {
+				Transaction:    &transaction.Transaction{},
+				ExecutionOrder: 0,
+			},
+		},
 	}
 	err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, map[string]*alteredAccount.AlteredAccount{}, testNumOfShards))
 	require.Nil(t, err)
 
+	ids = []string{logID}
 	err = esClient.DoMultiGet(context.Background(), ids, indexerdata.LogsIndex, true, genericResponse)
 	require.Nil(t, err)
 	require.JSONEq(t,
@@ -147,6 +198,7 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) {
 		MiniBlockHeaders: []dataBlock.MiniBlockHeader{
 			{},
 		},
+		ShardID: 1,
 	}
 	body = &dataBlock.Body{
 		MiniBlocks: []*dataBlock.MiniBlock{
@@ -163,4 +215,11 @@ func TestIndexLogSourceShardAndAfterDestinationAndAgainSource(t *testing.T) {
 	require.Nil(t, err)
 
 	require.False(t, genericResponse.Docs[0].Found)
+
+	ids = []string{event2ID, event3ID}
+	err = esClient.DoMultiGet(context.Background(), ids, indexerdata.EventsIndex, true, genericResponse)
+	require.Nil(t, err)
+
+	require.False(t, genericResponse.Docs[0].Found)
+	require.False(t, genericResponse.Docs[1].Found)
 }

integrationtests/testdata/logsCrossShard/event-do-something.json

@@ -0,0 +1,14 @@
+{
+  "logAddress": "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99",
+  "identifier": "do-something",
+  "address": "erd1w7jyzuj6cv4ngw8luhlkakatjpmjh3ql95lmxphd3vssc4vpymks6k5th7",
+  "topics": [
+    "746f70696331",
+    "746f70696332"
+  ],
+  "shardID": 1,
+  "txHash": "63726f73732d6c6f67",
+  "order": 1,
+  "timestamp": 6040,
+  "txOrder": 0
+}

integrationtests/testdata/logsCrossShard/event-transfer-destination.json

@@ -0,0 +1,15 @@
+{
+  "logAddress": "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99",
+  "identifier": "ESDTTransfer",
+  "address": "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99",
+  "topics": [
+    "455344542d61626364",
+    "",
+    "01"
+  ],
+  "shardID": 1,
+  "txHash": "63726f73732d6c6f67",
+  "order": 0,
+  "timestamp": 6040,
+  "txOrder": 0
+}

integrationtests/testdata/logsCrossShard/event-transfer-source-first.json

@@ -0,0 +1,15 @@
+{
+  "logAddress": "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99",
+  "identifier": "ESDTTransfer",
+  "address": "erd1ju8pkvg57cwdmjsjx58jlmnuf4l9yspstrhr9tgsrt98n9edpm2qtlgy99",
+  "topics": [
+    "455344542d61626364",
+    "",
+    "01"
+  ],
+  "shardID": 0,
+  "txOrder": 0,
+  "txHash": "63726f73732d6c6f67",
+  "order": 0,
+  "timestamp": 5040
+}

integrationtests/utils.go

@@ -57,7 +57,7 @@ func CreateElasticProcessor(
 		ValidatorPubkeyConverter: mock.NewPubkeyConverterMock(32),
 		DBClient:                 esClient,
 		EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex,
-			dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex,
+			dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex,
 			dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex},
 		Denomination: 18,
 	}

process/dataindexer/constants.go

@@ -47,6 +47,8 @@ const (
 	ESDTsIndex = "esdts"
 	// ValuesIndex is the Elasticsearch index for extra indexer information
 	ValuesIndex = "values"
+	// EventsIndex is the Elasticsearch index for log events
+	EventsIndex = "events"
 
 	// TransactionsPolicy is the Elasticsearch policy for the transactions
 	TransactionsPolicy = "transactions_policy"