From 92d6b3033707efeefa178b8ac8883f22df1e74cb Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 8 Jul 2024 11:49:25 +0300 Subject: [PATCH 1/6] relayed v3 integration --- client/elasticClient.go | 17 +++ data/transaction.go | 94 ++++++------- go.mod | 2 +- go.sum | 4 +- integrationtests/relayedTxV3_test.go | 99 ++++++++++++++ .../testdata/relayedTxV3/relayed-tx-v3.json | 54 ++++++++ process/elasticproc/elasticProcessor.go | 17 ++- .../factory/elasticProcessorFactory.go | 5 + process/elasticproc/interface.go | 1 + .../templatesAndPolicies/interface.go | 7 +- .../templatesAndPolicies/noKibana.go | 18 +++ .../templatesAndPolicies/withKibana.go | 5 + .../transactions/transactionDBBuilder.go | 73 +++++++++- .../transactions/transactionDBBuilder_test.go | 127 ++++++++++++++++++ templates/noKibana/innerTx.go | 63 +++++++++ templates/types.go | 6 + 16 files changed, 535 insertions(+), 57 deletions(-) create mode 100644 integrationtests/relayedTxV3_test.go create mode 100644 integrationtests/testdata/relayedTxV3/relayed-tx-v3.json create mode 100644 templates/noKibana/innerTx.go diff --git a/client/elasticClient.go b/client/elasticClient.go index 64f7e30f..5ae130a4 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -3,6 +3,7 @@ package client import ( "bytes" "context" + "errors" "fmt" "io" "net/http" @@ -84,6 +85,22 @@ func (ec *elasticClient) CheckAndCreateIndex(indexName string) error { return ec.createIndex(indexName) } +func (ec *elasticClient) PutMappings(indexName string, mappings *bytes.Buffer) error { + res, err := ec.client.Indices.PutMapping( + mappings, + ec.client.Indices.PutMapping.WithIndex(indexName), + ) + if err != nil { + return err + } + + if res.IsError() { + return errors.New(res.String()) + } + + return nil +} + // CheckAndCreateAlias creates a new alias if it does not already exist func (ec *elasticClient) CheckAndCreateAlias(alias string, indexName string) error { if ec.aliasExists(alias) { diff --git a/data/transaction.go b/data/transaction.go index d5cefd56..2a85ce52 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -2,57 +2,61 @@ package data import ( "time" + + "github.com/multiversx/mx-chain-core-go/data/transaction" ) // Transaction is a structure containing all the fields that need // to be saved for a transaction. It has all the default fields // plus some extra information for ease of search and filter type Transaction struct { - MBHash string `json:"miniBlockHash"` - Nonce uint64 `json:"nonce"` - Round uint64 `json:"round"` - Value string `json:"value"` - ValueNum float64 `json:"valueNum"` - Receiver string `json:"receiver"` - Sender string `json:"sender"` - ReceiverShard uint32 `json:"receiverShard"` - SenderShard uint32 `json:"senderShard"` - GasPrice uint64 `json:"gasPrice"` - GasLimit uint64 `json:"gasLimit"` - GasUsed uint64 `json:"gasUsed"` - Fee string `json:"fee"` - FeeNum float64 `json:"feeNum"` - InitialPaidFee string `json:"initialPaidFee,omitempty"` - Data []byte `json:"data"` - Signature string `json:"signature"` - Timestamp time.Duration `json:"timestamp"` - Status string `json:"status"` - SearchOrder uint32 `json:"searchOrder"` - SenderUserName []byte `json:"senderUserName,omitempty"` - ReceiverUserName []byte `json:"receiverUserName,omitempty"` - HasSCR bool `json:"hasScResults,omitempty"` - IsScCall bool `json:"isScCall,omitempty"` - HasOperations bool `json:"hasOperations,omitempty"` - HasLogs bool `json:"hasLogs,omitempty"` - Tokens []string `json:"tokens,omitempty"` - ESDTValues []string `json:"esdtValues,omitempty"` - ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"` - Receivers []string `json:"receivers,omitempty"` - ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"` - Type string `json:"type,omitempty"` - Operation string `json:"operation,omitempty"` - Function string `json:"function,omitempty"` - IsRelayed bool `json:"isRelayed,omitempty"` - Version uint32 `json:"version,omitempty"` - GuardianAddress string `json:"guardian,omitempty"` - GuardianSignature string `json:"guardianSignature,omitempty"` - ErrorEvent bool `json:"errorEvent,omitempty"` - CompletedEvent bool `json:"completedEvent,omitempty"` - ExecutionOrder int `json:"-"` - SmartContractResults []*ScResult `json:"-"` - Hash string `json:"-"` - BlockHash string `json:"-"` - HadRefund bool `json:"-"` + MBHash string `json:"miniBlockHash"` + Nonce uint64 `json:"nonce"` + Round uint64 `json:"round"` + Value string `json:"value"` + ValueNum float64 `json:"valueNum"` + Receiver string `json:"receiver"` + Sender string `json:"sender"` + ReceiverShard uint32 `json:"receiverShard"` + SenderShard uint32 `json:"senderShard"` + GasPrice uint64 `json:"gasPrice"` + GasLimit uint64 `json:"gasLimit"` + GasUsed uint64 `json:"gasUsed"` + Fee string `json:"fee"` + FeeNum float64 `json:"feeNum"` + InitialPaidFee string `json:"initialPaidFee,omitempty"` + Data []byte `json:"data"` + Signature string `json:"signature"` + Timestamp time.Duration `json:"timestamp"` + Status string `json:"status"` + SearchOrder uint32 `json:"searchOrder"` + SenderUserName []byte `json:"senderUserName,omitempty"` + ReceiverUserName []byte `json:"receiverUserName,omitempty"` + HasSCR bool `json:"hasScResults,omitempty"` + IsScCall bool `json:"isScCall,omitempty"` + HasOperations bool `json:"hasOperations,omitempty"` + HasLogs bool `json:"hasLogs,omitempty"` + Tokens []string `json:"tokens,omitempty"` + ESDTValues []string `json:"esdtValues,omitempty"` + ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"` + Receivers []string `json:"receivers,omitempty"` + ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"` + Type string `json:"type,omitempty"` + Operation string `json:"operation,omitempty"` + Function string `json:"function,omitempty"` + IsRelayed bool `json:"isRelayed,omitempty"` + Version uint32 `json:"version,omitempty"` + GuardianAddress string `json:"guardian,omitempty"` + GuardianSignature string `json:"guardianSignature,omitempty"` + ErrorEvent bool `json:"errorEvent,omitempty"` + CompletedEvent bool `json:"completedEvent,omitempty"` + RelayedAddr string `json:"relayed,omitempty"` + InnerTransactions []*transaction.FrontendTransaction `json:"innerTransactions,omitempty"` + ExecutionOrder int `json:"-"` + SmartContractResults []*ScResult `json:"-"` + Hash string `json:"-"` + BlockHash string `json:"-"` + HadRefund bool `json:"-"` } // Receipt is a structure containing all the fields that need to be safe for a Receipt diff --git a/go.mod b/go.mod index 46f09b26..312a16e7 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e - github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 + github.com/multiversx/mx-chain-core-go v1.2.21-0.20240703140829-626328c91a8d github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 github.com/multiversx/mx-chain-vm-common-go v1.5.13-0.20240619122724-2bd2e64cebdc github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index ad53f92a..975ffc74 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e h1:Tsmwhu+UleE+l3buPuqXSKTqfu5FbPmzQ4MjMoUvCWA= github.com/multiversx/mx-chain-communication-go v1.0.15-0.20240508074652-e128a1c05c8e/go.mod h1:2yXl18wUbuV3cRZr7VHxM1xo73kTaC1WUcu2kx8R034= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840 h1:2mCrTUmbbA+Xv4UifZY9xptrGjcJBcJ2wavSb4FwejU= -github.com/multiversx/mx-chain-core-go v1.2.21-0.20240508071047-fefea5737840/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240703140829-626328c91a8d h1:2x1arnxYt28ZlDAZj61dzmG4NqoUmAZbe3pTFsBZHek= +github.com/multiversx/mx-chain-core-go v1.2.21-0.20240703140829-626328c91a8d/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12-0.20240508074452-cc21c1b505df h1:clihfi78bMEOWk/qw6WA4uQbCM2e2NGliqswLAvw19k= github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57 h1:g9t410dqjcb7UUptbVd/H6Ua12sEzWU4v7VplyNvRZ0= github.com/multiversx/mx-chain-logger-go v1.0.15-0.20240508072523-3f00a726af57/go.mod h1:cY6CIXpndW5g5PTPn4WzPwka/UBEf+mgw+PSY5pHGAU= diff --git a/integrationtests/relayedTxV3_test.go b/integrationtests/relayedTxV3_test.go new file mode 100644 index 00000000..d492c753 --- /dev/null +++ b/integrationtests/relayedTxV3_test.go @@ -0,0 +1,99 @@ +//go:build integrationtests + +package integrationtests + +import ( + "context" + "encoding/hex" + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-core-go/data/transaction" + indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/stretchr/testify/require" + "math/big" + "testing" +) + +func TestRelayedTxV3(t *testing.T) { + setLogLevelDebug() + + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + txHash := []byte("relayedTxV3") + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + } + + body := &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.TxBlock, + SenderShardID: 0, + ReceiverShardID: 0, + TxHashes: [][]byte{txHash}, + }, + }, + } + + address1 := "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u" + address2 := "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht" + initialTx := &transaction.Transaction{ + Nonce: 1000, + SndAddr: decodeAddress(address1), + RcvAddr: decodeAddress(address2), + GasLimit: 15406000, + GasPrice: 1000000000, + Value: big.NewInt(0), + InnerTransactions: []*transaction.Transaction{ + { + Nonce: 10, + SndAddr: decodeAddress(address1), + RcvAddr: decodeAddress(address2), + GasLimit: 15406000, + GasPrice: 1000000000, + Value: big.NewInt(0), + }, + { + Nonce: 20, + SndAddr: decodeAddress(address1), + RcvAddr: decodeAddress(address2), + GasLimit: 15406000, + GasPrice: 1000000000, + Value: big.NewInt(1000), + }, + }, + } + + txInfo := &outport.TxInfo{ + Transaction: initialTx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 10556000, + Fee: big.NewInt(2257820000000000), + InitialPaidFee: big.NewInt(2306320000000000), + }, + ExecutionOrder: 0, + } + + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids := []string{hex.EncodeToString(txHash)} + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/relayedTxV3/relayed-tx-v3.json"), + string(genericResponse.Docs[0].Source), + ) +} diff --git a/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json b/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json new file mode 100644 index 00000000..62a67fa1 --- /dev/null +++ b/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json @@ -0,0 +1,54 @@ +{ + "miniBlockHash": "b3222be4506429fa237393c56b732a283eea5ccab465deb66b6348083793b6c9", + "nonce": 1000, + "round": 50, + "value": "0", + "valueNum": 0, + "receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", + "sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 15406000, + "gasUsed": 10556000, + "fee": "2257820000000000", + "feeNum": 0.00225782, + "initialPaidFee": "2306320000000000", + "data": null, + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "receivers": [ + "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", + "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht" + ], + "receiversShardIDs": [ + 1, + 1 + ], + "operation": "transfer", + "isRelayed": true, + "innerTransactions": [ + { + "nonce": 10, + "value": "0", + "receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", + "sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u", + "gasPrice": 1000000000, + "gasLimit": 15406000, + "chainID": "", + "version": 0 + }, + { + "nonce": 20, + "value": "1000", + "receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", + "sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u", + "gasPrice": 1000000000, + "gasLimit": 15406000, + "chainID": "", + "version": 0 + } + ] +} \ No newline at end of file diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 640e5f09..89634a95 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -20,6 +20,7 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" + "github.com/multiversx/mx-chain-es-indexer-go/templates" logger "github.com/multiversx/mx-chain-logger-go" ) @@ -44,6 +45,7 @@ type ArgElasticProcessor struct { ImportDB bool IndexTemplates map[string]*bytes.Buffer IndexPolicies map[string]*bytes.Buffer + ExtraMappings []templates.ExtraMapping EnabledIndexes map[string]struct{} TransactionsProc DBTransactionsHandler AccountsProc DBAccountHandler @@ -94,7 +96,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err bulkRequestMaxSize: arguments.BulkRequestMaxSize, } - err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies) + err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings) if err != nil { return nil, err } @@ -105,7 +107,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err } // TODO move all the index create part in a new component -func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*bytes.Buffer) error { +func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*bytes.Buffer, extraMappings []templates.ExtraMapping) error { err := ei.createOpenDistroTemplates(indexTemplates) if err != nil { return err @@ -135,6 +137,17 @@ func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*b return err } + return ei.addExtraMappings(extraMappings) +} + +func (ei *elasticProcessor) addExtraMappings(extraMappings []templates.ExtraMapping) error { + for _, mappingsTuple := range extraMappings { + err := ei.elasticClient.PutMappings(mappingsTuple.Index, mappingsTuple.Mappings) + if err != nil { + return err + } + } + return nil } diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index eb10110c..ad865953 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -40,6 +40,10 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E if err != nil { return nil, err } + extraMappings, err := templatesAndPoliciesReader.GetExtraMappings() + if err != nil { + return nil, err + } enabledIndexesMap := make(map[string]struct{}) for _, index := range arguments.EnabledIndexes { @@ -119,6 +123,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E UseKibana: arguments.UseKibana, IndexTemplates: indexTemplates, IndexPolicies: indexPolicies, + ExtraMappings: extraMappings, OperationsProc: operationsProc, ImportDB: arguments.ImportDB, Version: arguments.Version, diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 997697b2..fe4b565f 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -21,6 +21,7 @@ type DatabaseClientHandler interface { DoCountRequest(ctx context.Context, index string, body []byte) (uint64, error) UpdateByQuery(ctx context.Context, index string, buff *bytes.Buffer) error + PutMappings(indexName string, mappings *bytes.Buffer) error CheckAndCreateIndex(index string) error CheckAndCreateAlias(alias string, index string) error CheckAndCreateTemplate(templateName string, template *bytes.Buffer) error diff --git a/process/elasticproc/templatesAndPolicies/interface.go b/process/elasticproc/templatesAndPolicies/interface.go index f4b740c9..2215c781 100644 --- a/process/elasticproc/templatesAndPolicies/interface.go +++ b/process/elasticproc/templatesAndPolicies/interface.go @@ -1,8 +1,13 @@ package templatesAndPolicies -import "bytes" +import ( + "bytes" + + "github.com/multiversx/mx-chain-es-indexer-go/templates" +) // TemplatesAndPoliciesHandler defines the actions that a templates and policies handler should do type TemplatesAndPoliciesHandler interface { GetElasticTemplatesAndPolicies() (map[string]*bytes.Buffer, map[string]*bytes.Buffer, error) + GetExtraMappings() ([]templates.ExtraMapping, error) } diff --git a/process/elasticproc/templatesAndPolicies/noKibana.go b/process/elasticproc/templatesAndPolicies/noKibana.go index a6928858..d87acaba 100644 --- a/process/elasticproc/templatesAndPolicies/noKibana.go +++ b/process/elasticproc/templatesAndPolicies/noKibana.go @@ -4,6 +4,7 @@ import ( "bytes" indexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/templates" "github.com/multiversx/mx-chain-es-indexer-go/templates/noKibana" ) @@ -45,3 +46,20 @@ func (tr *templatesAndPolicyReaderNoKibana) GetElasticTemplatesAndPolicies() (ma return indexTemplates, indexPolicies, nil } + +// GetExtraMappings will return an array of indices extra mappings +func (tr *templatesAndPolicyReaderNoKibana) GetExtraMappings() ([]templates.ExtraMapping, error) { + transactionsExtraMappings := templates.ExtraMapping{ + Index: indexer.TransactionsIndex, + Mappings: noKibana.InnerTxs.ToBuffer(), + } + operationsExtraMappings := templates.ExtraMapping{ + Index: indexer.OperationsIndex, + Mappings: noKibana.InnerTxs.ToBuffer(), + } + + return []templates.ExtraMapping{ + transactionsExtraMappings, + operationsExtraMappings, + }, nil +} diff --git a/process/elasticproc/templatesAndPolicies/withKibana.go b/process/elasticproc/templatesAndPolicies/withKibana.go index 2fe1cd38..33ddaa23 100644 --- a/process/elasticproc/templatesAndPolicies/withKibana.go +++ b/process/elasticproc/templatesAndPolicies/withKibana.go @@ -4,6 +4,7 @@ import ( "bytes" indexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/multiversx/mx-chain-es-indexer-go/templates" "github.com/multiversx/mx-chain-es-indexer-go/templates/withKibana" ) @@ -22,6 +23,10 @@ func (tr *templatesAndPolicyReaderWithKibana) GetElasticTemplatesAndPolicies() ( return indexTemplates, indexPolicies, nil } +func (tr *templatesAndPolicyReaderWithKibana) GetExtraMappings() ([]templates.ExtraMapping, error) { + return []templates.ExtraMapping{}, nil +} + func getTemplatesKibana() map[string]*bytes.Buffer { indexTemplates := make(map[string]*bytes.Buffer) diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 0b497ef7..ac5818d4 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" + "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" @@ -86,7 +87,8 @@ func (dtb *dbTransactionBuilder) prepareTransaction( senderUserName := converters.TruncateFieldIfExceedsMaxLengthBase64(string(tx.SndUserName)) receiverUserName := converters.TruncateFieldIfExceedsMaxLengthBase64(string(tx.RcvUserName)) - return &data.Transaction{ + + eTx := &data.Transaction{ Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: tx.Nonce, @@ -110,19 +112,78 @@ func (dtb *dbTransactionBuilder) prepareTransaction( ReceiverUserName: []byte(receiverUserName), SenderUserName: []byte(senderUserName), IsScCall: isScCall, - Operation: res.Operation, - Function: converters.TruncateFieldIfExceedsMaxLength(res.Function), ESDTValues: esdtValues, ESDTValuesNum: esdtValuesNum, - Tokens: converters.TruncateSliceElementsIfExceedsMaxLength(res.Tokens), Receivers: receiversAddr, - ReceiversShardIDs: res.ReceiversShardID, - IsRelayed: res.IsRelayed, Version: tx.Version, GuardianAddress: guardianAddress, GuardianSignature: hex.EncodeToString(tx.GuardianSignature), ExecutionOrder: int(txInfo.ExecutionOrder), } + + isRelayedV3 := len(tx.InnerTransactions) > 0 + if isRelayedV3 { + eTx.Operation = res.Operation + dtb.addRelayedV3InfoInIndexerTx(tx, eTx, numOfShards) + + return eTx + } + + eTx.Operation = res.Operation + eTx.Function = converters.TruncateFieldIfExceedsMaxLength(res.Function) + eTx.Tokens = converters.TruncateSliceElementsIfExceedsMaxLength(res.Tokens) + eTx.ReceiversShardIDs = res.ReceiversShardID + eTx.IsRelayed = res.IsRelayed + + return eTx +} + +func (dtb *dbTransactionBuilder) addRelayedV3InfoInIndexerTx(tx *transaction.Transaction, indexerTx *data.Transaction, numOfShards uint32) { + if len(tx.InnerTransactions) == 0 { + return + } + + innerTxs := make([]*transaction.FrontendTransaction, 0, len(tx.InnerTransactions)) + receivers := make([]string, 0, len(tx.InnerTransactions)) + receiversShardIDs := make([]uint32, 0, len(tx.InnerTransactions)) + for _, innerTx := range tx.InnerTransactions { + frontEndTx := &transaction.FrontendTransaction{ + Nonce: innerTx.Nonce, + Value: innerTx.Value.String(), + Receiver: dtb.addressPubkeyConverter.SilentEncode(innerTx.RcvAddr, log), + Sender: dtb.addressPubkeyConverter.SilentEncode(innerTx.SndAddr, log), + SenderUsername: innerTx.SndUserName, + ReceiverUsername: innerTx.RcvUserName, + GasPrice: innerTx.GasPrice, + GasLimit: innerTx.GasLimit, + Data: innerTx.Data, + Signature: hex.EncodeToString(innerTx.Signature), + ChainID: string(innerTx.ChainID), + Version: innerTx.Version, + Options: innerTx.Options, + } + + if len(innerTx.GuardianAddr) > 0 { + frontEndTx.GuardianAddr = dtb.addressPubkeyConverter.SilentEncode(innerTx.GuardianAddr, log) + frontEndTx.GuardianSignature = hex.EncodeToString(innerTx.GuardianSignature) + } + + if len(innerTx.RelayerAddr) > 0 { + frontEndTx.Relayer = dtb.addressPubkeyConverter.SilentEncode(innerTx.RelayerAddr, log) + } + + receivers = append(receivers, frontEndTx.Receiver) + receiversShardIDs = append(receiversShardIDs, sharding.ComputeShardID(innerTx.RcvAddr, numOfShards)) + + innerTxs = append(innerTxs, frontEndTx) + } + + indexerTx.InnerTransactions = innerTxs + indexerTx.IsRelayed = true + indexerTx.Receivers = receivers + indexerTx.ReceiversShardIDs = receiversShardIDs + + return } func (dtb *dbTransactionBuilder) prepareRewardTransaction( diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index ceeca3de..b892166e 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -204,3 +204,130 @@ func TestGetMoveBalanceTransactionInvalid(t *testing.T) { dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, header, status, 3) require.Equal(t, expectedTx, dbTx) } + +func TestTxsDatabaseProcessor_PrepareTransactionRelayedV3(t *testing.T) { + t.Parallel() + + txHash := []byte("txHash") + mbHash := []byte("mbHash") + mb := &block.MiniBlock{TxHashes: [][]byte{txHash}} + header := &block.Header{Nonce: 2} + status := "Success" + gasPrice := uint64(1000) + gasLimit := uint64(1000) + cp := createCommonProcessor() + + tx := &transaction.Transaction{ + Nonce: 1, + Value: big.NewInt(1000), + RcvAddr: []byte("receiver"), + SndAddr: []byte("sender"), + GasPrice: gasPrice, + GasLimit: gasLimit, + ChainID: []byte("1"), + Version: 1, + Signature: []byte("signature"), + RcvUserName: []byte("rcv"), + SndUserName: []byte("snd"), + InnerTransactions: []*transaction.Transaction{ + { + Nonce: 2, + Value: big.NewInt(1000), + SndAddr: []byte("inner1"), + RcvAddr: []byte("rcv1"), + Data: []byte("data1"), + Signature: []byte("signature1"), + GasPrice: gasPrice, + GasLimit: gasLimit, + ChainID: []byte("1"), + Version: 1, + }, + { + Nonce: 3, + Value: big.NewInt(1), + SndAddr: []byte("inner2"), + RcvAddr: []byte("rcv2"), + Data: []byte("data2"), + Signature: []byte("signature2"), + GasPrice: gasPrice, + GasLimit: gasLimit, + ChainID: []byte("1"), + Version: 1, + }, + }, + } + + txInfo := &outport.TxInfo{ + Transaction: tx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 500, + Fee: big.NewInt(100), + InitialPaidFee: big.NewInt(100), + }, + ExecutionOrder: 0, + } + + senderAddr, err := cp.addressPubkeyConverter.Encode(tx.RcvAddr) + require.Nil(t, err) + receiverAddr, err := cp.addressPubkeyConverter.Encode(tx.SndAddr) + require.Nil(t, err) + + expectedTx := &data.Transaction{ + Hash: hex.EncodeToString(txHash), + MBHash: hex.EncodeToString(mbHash), + Nonce: tx.Nonce, + Round: header.Round, + Value: tx.Value.String(), + ValueNum: 1e-15, + Receiver: senderAddr, + Sender: receiverAddr, + ReceiverShard: mb.ReceiverShardID, + SenderShard: mb.SenderShardID, + GasPrice: gasPrice, + GasLimit: gasLimit, + GasUsed: uint64(500), + InitialPaidFee: "100", + Signature: hex.EncodeToString(tx.Signature), + Timestamp: time.Duration(header.GetTimeStamp()), + Status: status, + Fee: "100", + FeeNum: 1e-16, + ReceiverUserName: []byte("rcv"), + SenderUserName: []byte("snd"), + ESDTValuesNum: []float64{}, + Operation: "transfer", + Version: 1, + Receivers: []string{"72637631", "72637632"}, + ReceiversShardIDs: []uint32{1, 2}, + InnerTransactions: []*transaction.FrontendTransaction{ + { + Nonce: 2, + Value: "1000", + Sender: "696e6e657231", + Receiver: "72637631", + GasPrice: gasPrice, + GasLimit: gasLimit, + Signature: "7369676e617475726531", + Data: []byte("data1"), + Version: 1, + ChainID: "1", + }, + { + Nonce: 3, + Value: "1", + Sender: "696e6e657232", + Receiver: "72637632", + GasPrice: gasPrice, + GasLimit: gasLimit, + Signature: "7369676e617475726532", + Data: []byte("data2"), + Version: 1, + ChainID: "1", + }, + }, + IsRelayed: true, + } + + dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, header, status, 3) + require.Equal(t, expectedTx, dbTx) +} diff --git a/templates/noKibana/innerTx.go b/templates/noKibana/innerTx.go new file mode 100644 index 00000000..c97f5cbe --- /dev/null +++ b/templates/noKibana/innerTx.go @@ -0,0 +1,63 @@ +package noKibana + +// InnerTxs will hold the configuration for the inner transaction structure +var InnerTxs = Object{ + "properties": Object{ + "innerTransactions": Object{ + "properties": Object{ + "nonce": Object{ + "type": "double", + }, + "value": Object{ + "type": "keyword", + }, + "sender": Object{ + "type": "keyword", + }, + "receiver": Object{ + "type": "keyword", + }, + "senderUserName": Object{ + "type": "keyword", + }, + "receiverUsername": Object{ + "type": "keyword", + }, + "gasLimit": Object{ + "index": "false", + "type": "double", + }, + "gasPrice": Object{ + "index": "false", + "type": "double", + }, + "data": Object{ + "type": "text", + }, + "signature": Object{ + "index": "false", + "type": "keyword", + }, + "chainID": Object{ + "type": "keyword", + }, + "version": Object{ + "type": "long", + }, + "options": Object{ + "type": "long", + }, + "guardian": Object{ + "type": "keyword", + }, + "guardianSignature": Object{ + "index": "false", + "type": "keyword", + }, + "relayer": Object{ + "type": "keyword", + }, + }, + }, + }, +} diff --git a/templates/types.go b/templates/types.go index 74371fb5..3d776138 100644 --- a/templates/types.go +++ b/templates/types.go @@ -5,6 +5,12 @@ import ( "encoding/json" ) +// ExtraMapping holds the tuple for the index and extra mappings +type ExtraMapping struct { + Index string + Mappings *bytes.Buffer +} + // Array type will rename type []interface{} type Array []interface{} From f59580a961ca53b9b9dfe5de60fd0db6f69ef57d Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 8 Jul 2024 11:57:29 +0300 Subject: [PATCH 2/6] fix unit tests --- mock/databaseWriterStub.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mock/databaseWriterStub.go b/mock/databaseWriterStub.go index 9a41e0ff..1df608a8 100644 --- a/mock/databaseWriterStub.go +++ b/mock/databaseWriterStub.go @@ -14,6 +14,11 @@ type DatabaseWriterStub struct { DoScrollRequestCalled func(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error) error } +// PutMappings - +func (dwm *DatabaseWriterStub) PutMappings(_ string, _ *bytes.Buffer) error { + return nil +} + // UpdateByQuery - func (dwm *DatabaseWriterStub) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error { return nil From e7f4007f995a42aa61363e2122793075d03928a8 Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 8 Jul 2024 11:59:33 +0300 Subject: [PATCH 3/6] fix linter issues --- process/elasticproc/transactions/transactionDBBuilder.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index ac5818d4..aa9808eb 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -182,8 +182,6 @@ func (dtb *dbTransactionBuilder) addRelayedV3InfoInIndexerTx(tx *transaction.Tra indexerTx.IsRelayed = true indexerTx.Receivers = receivers indexerTx.ReceiversShardIDs = receiversShardIDs - - return } func (dtb *dbTransactionBuilder) prepareRewardTransaction( From e632bc2407a4dfdc076e74bfc1bd6f59cb3d4d63 Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 8 Jul 2024 12:04:53 +0300 Subject: [PATCH 4/6] new mappings in transaction and operation index --- templates/noKibana/operations.go | 56 ++++++++++++++++++++++++++++++ templates/noKibana/transactions.go | 56 ++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/templates/noKibana/operations.go b/templates/noKibana/operations.go index 8c734d9d..9834f19b 100644 --- a/templates/noKibana/operations.go +++ b/templates/noKibana/operations.go @@ -169,6 +169,62 @@ var Operations = Object{ "index": "false", "type": "keyword", }, + "innerTransactions": Object{ + "properties": Object{ + "nonce": Object{ + "type": "double", + }, + "value": Object{ + "type": "keyword", + }, + "sender": Object{ + "type": "keyword", + }, + "receiver": Object{ + "type": "keyword", + }, + "senderUserName": Object{ + "type": "keyword", + }, + "receiverUsername": Object{ + "type": "keyword", + }, + "gasLimit": Object{ + "index": "false", + "type": "double", + }, + "gasPrice": Object{ + "index": "false", + "type": "double", + }, + "data": Object{ + "type": "text", + }, + "signature": Object{ + "index": "false", + "type": "keyword", + }, + "chainID": Object{ + "type": "keyword", + }, + "version": Object{ + "type": "long", + }, + "options": Object{ + "type": "long", + }, + "guardian": Object{ + "type": "keyword", + }, + "guardianSignature": Object{ + "index": "false", + "type": "keyword", + }, + "relayer": Object{ + "type": "keyword", + }, + }, + }, }, }, } diff --git a/templates/noKibana/transactions.go b/templates/noKibana/transactions.go index 91ebea75..16ddef08 100644 --- a/templates/noKibana/transactions.go +++ b/templates/noKibana/transactions.go @@ -135,6 +135,62 @@ var Transactions = Object{ "index": "false", "type": "keyword", }, + "innerTransactions": Object{ + "properties": Object{ + "nonce": Object{ + "type": "double", + }, + "value": Object{ + "type": "keyword", + }, + "sender": Object{ + "type": "keyword", + }, + "receiver": Object{ + "type": "keyword", + }, + "senderUserName": Object{ + "type": "keyword", + }, + "receiverUsername": Object{ + "type": "keyword", + }, + "gasLimit": Object{ + "index": "false", + "type": "double", + }, + "gasPrice": Object{ + "index": "false", + "type": "double", + }, + "data": Object{ + "type": "text", + }, + "signature": Object{ + "index": "false", + "type": "keyword", + }, + "chainID": Object{ + "type": "keyword", + }, + "version": Object{ + "type": "long", + }, + "options": Object{ + "type": "long", + }, + "guardian": Object{ + "type": "keyword", + }, + "guardianSignature": Object{ + "index": "false", + "type": "keyword", + }, + "relayer": Object{ + "type": "keyword", + }, + }, + }, }, }, } From 643032ac245a6336a579dfa8f4e23d6d66214c67 Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 8 Jul 2024 12:11:28 +0300 Subject: [PATCH 5/6] comments --- client/elasticClient.go | 1 + process/elasticproc/templatesAndPolicies/withKibana.go | 1 + 2 files changed, 2 insertions(+) diff --git a/client/elasticClient.go b/client/elasticClient.go index 5ae130a4..f388c049 100644 --- a/client/elasticClient.go +++ b/client/elasticClient.go @@ -85,6 +85,7 @@ func (ec *elasticClient) CheckAndCreateIndex(indexName string) error { return ec.createIndex(indexName) } +// PutMappings will put the provided mappings to a given index func (ec *elasticClient) PutMappings(indexName string, mappings *bytes.Buffer) error { res, err := ec.client.Indices.PutMapping( mappings, diff --git a/process/elasticproc/templatesAndPolicies/withKibana.go b/process/elasticproc/templatesAndPolicies/withKibana.go index 33ddaa23..7ffe6b2f 100644 --- a/process/elasticproc/templatesAndPolicies/withKibana.go +++ b/process/elasticproc/templatesAndPolicies/withKibana.go @@ -23,6 +23,7 @@ func (tr *templatesAndPolicyReaderWithKibana) GetElasticTemplatesAndPolicies() ( return indexTemplates, indexPolicies, nil } +// GetExtraMappings will return an array of indices extra mappings func (tr *templatesAndPolicyReaderWithKibana) GetExtraMappings() ([]templates.ExtraMapping, error) { return []templates.ExtraMapping{}, nil } From 98808ec1d4da3146098ba2f6d1a5a860fa03eca2 Mon Sep 17 00:00:00 2001 From: miiu Date: Tue, 16 Jul 2024 15:27:46 +0300 Subject: [PATCH 6/6] fixes after review --- integrationtests/testdata/relayedTxV3/relayed-tx-v3.json | 2 +- process/elasticproc/transactions/transactionDBBuilder.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json b/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json index 62a67fa1..210df682 100644 --- a/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json +++ b/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json @@ -51,4 +51,4 @@ "version": 0 } ] -} \ No newline at end of file +} diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index aa9808eb..f22e1a31 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -119,17 +119,16 @@ func (dtb *dbTransactionBuilder) prepareTransaction( GuardianAddress: guardianAddress, GuardianSignature: hex.EncodeToString(tx.GuardianSignature), ExecutionOrder: int(txInfo.ExecutionOrder), + Operation: res.Operation, } isRelayedV3 := len(tx.InnerTransactions) > 0 if isRelayedV3 { - eTx.Operation = res.Operation dtb.addRelayedV3InfoInIndexerTx(tx, eTx, numOfShards) return eTx } - eTx.Operation = res.Operation eTx.Function = converters.TruncateFieldIfExceedsMaxLength(res.Function) eTx.Tokens = converters.TruncateSliceElementsIfExceedsMaxLength(res.Tokens) eTx.ReceiversShardIDs = res.ReceiversShardID