Skip to content

Commit

Permalink
Merge branch 'rc/v1.7.next1' into update_deps
Browse files Browse the repository at this point in the history
  • Loading branch information
sstanculeanu authored Jul 29, 2024
2 parents eaa70a3 + 9f42664 commit 51bea3e
Show file tree
Hide file tree
Showing 17 changed files with 648 additions and 54 deletions.
18 changes: 18 additions & 0 deletions client/elasticClient.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package client
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -84,6 +85,23 @@ func (ec *elasticClient) CheckAndCreateIndex(indexName string) error {
return ec.createIndex(indexName)
}

// PutMappings will put the provided mappings to a given index
func (ec *elasticClient) PutMappings(indexName string, mappings *bytes.Buffer) error {
res, err := ec.client.Indices.PutMapping(
mappings,
ec.client.Indices.PutMapping.WithIndex(indexName),
)
if err != nil {
return err
}

if res.IsError() {
return errors.New(res.String())
}

return nil
}

// CheckAndCreateAlias creates a new alias if it does not already exist
func (ec *elasticClient) CheckAndCreateAlias(alias string, indexName string) error {
if ec.aliasExists(alias) {
Expand Down
94 changes: 49 additions & 45 deletions data/transaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,57 +2,61 @@ package data

import (
"time"

"github.com/multiversx/mx-chain-core-go/data/transaction"
)

// Transaction is a structure containing all the fields that need
// to be saved for a transaction. It has all the default fields
// plus some extra information for ease of search and filter
type Transaction struct {
MBHash string `json:"miniBlockHash"`
Nonce uint64 `json:"nonce"`
Round uint64 `json:"round"`
Value string `json:"value"`
ValueNum float64 `json:"valueNum"`
Receiver string `json:"receiver"`
Sender string `json:"sender"`
ReceiverShard uint32 `json:"receiverShard"`
SenderShard uint32 `json:"senderShard"`
GasPrice uint64 `json:"gasPrice"`
GasLimit uint64 `json:"gasLimit"`
GasUsed uint64 `json:"gasUsed"`
Fee string `json:"fee"`
FeeNum float64 `json:"feeNum"`
InitialPaidFee string `json:"initialPaidFee,omitempty"`
Data []byte `json:"data"`
Signature string `json:"signature"`
Timestamp time.Duration `json:"timestamp"`
Status string `json:"status"`
SearchOrder uint32 `json:"searchOrder"`
SenderUserName []byte `json:"senderUserName,omitempty"`
ReceiverUserName []byte `json:"receiverUserName,omitempty"`
HasSCR bool `json:"hasScResults,omitempty"`
IsScCall bool `json:"isScCall,omitempty"`
HasOperations bool `json:"hasOperations,omitempty"`
HasLogs bool `json:"hasLogs,omitempty"`
Tokens []string `json:"tokens,omitempty"`
ESDTValues []string `json:"esdtValues,omitempty"`
ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"`
Receivers []string `json:"receivers,omitempty"`
ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"`
Type string `json:"type,omitempty"`
Operation string `json:"operation,omitempty"`
Function string `json:"function,omitempty"`
IsRelayed bool `json:"isRelayed,omitempty"`
Version uint32 `json:"version,omitempty"`
GuardianAddress string `json:"guardian,omitempty"`
GuardianSignature string `json:"guardianSignature,omitempty"`
ErrorEvent bool `json:"errorEvent,omitempty"`
CompletedEvent bool `json:"completedEvent,omitempty"`
ExecutionOrder int `json:"-"`
SmartContractResults []*ScResult `json:"-"`
Hash string `json:"-"`
BlockHash string `json:"-"`
HadRefund bool `json:"-"`
MBHash string `json:"miniBlockHash"`
Nonce uint64 `json:"nonce"`
Round uint64 `json:"round"`
Value string `json:"value"`
ValueNum float64 `json:"valueNum"`
Receiver string `json:"receiver"`
Sender string `json:"sender"`
ReceiverShard uint32 `json:"receiverShard"`
SenderShard uint32 `json:"senderShard"`
GasPrice uint64 `json:"gasPrice"`
GasLimit uint64 `json:"gasLimit"`
GasUsed uint64 `json:"gasUsed"`
Fee string `json:"fee"`
FeeNum float64 `json:"feeNum"`
InitialPaidFee string `json:"initialPaidFee,omitempty"`
Data []byte `json:"data"`
Signature string `json:"signature"`
Timestamp time.Duration `json:"timestamp"`
Status string `json:"status"`
SearchOrder uint32 `json:"searchOrder"`
SenderUserName []byte `json:"senderUserName,omitempty"`
ReceiverUserName []byte `json:"receiverUserName,omitempty"`
HasSCR bool `json:"hasScResults,omitempty"`
IsScCall bool `json:"isScCall,omitempty"`
HasOperations bool `json:"hasOperations,omitempty"`
HasLogs bool `json:"hasLogs,omitempty"`
Tokens []string `json:"tokens,omitempty"`
ESDTValues []string `json:"esdtValues,omitempty"`
ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"`
Receivers []string `json:"receivers,omitempty"`
ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"`
Type string `json:"type,omitempty"`
Operation string `json:"operation,omitempty"`
Function string `json:"function,omitempty"`
IsRelayed bool `json:"isRelayed,omitempty"`
Version uint32 `json:"version,omitempty"`
GuardianAddress string `json:"guardian,omitempty"`
GuardianSignature string `json:"guardianSignature,omitempty"`
ErrorEvent bool `json:"errorEvent,omitempty"`
CompletedEvent bool `json:"completedEvent,omitempty"`
RelayedAddr string `json:"relayed,omitempty"`
InnerTransactions []*transaction.FrontendTransaction `json:"innerTransactions,omitempty"`
ExecutionOrder int `json:"-"`
SmartContractResults []*ScResult `json:"-"`
Hash string `json:"-"`
BlockHash string `json:"-"`
HadRefund bool `json:"-"`
}

// Receipt is a structure containing all the fields that need to be safe for a Receipt
Expand Down
99 changes: 99 additions & 0 deletions integrationtests/relayedTxV3_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
//go:build integrationtests

package integrationtests

import (
"context"
"encoding/hex"
dataBlock "github.com/multiversx/mx-chain-core-go/data/block"
"github.com/multiversx/mx-chain-core-go/data/outport"
"github.com/multiversx/mx-chain-core-go/data/transaction"
indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
"github.com/stretchr/testify/require"
"math/big"
"testing"
)

func TestRelayedTxV3(t *testing.T) {
setLogLevelDebug()

esClient, err := createESClient(esURL)
require.Nil(t, err)

esProc, err := CreateElasticProcessor(esClient)
require.Nil(t, err)

txHash := []byte("relayedTxV3")
header := &dataBlock.Header{
Round: 50,
TimeStamp: 5040,
}

body := &dataBlock.Body{
MiniBlocks: dataBlock.MiniBlockSlice{
{
Type: dataBlock.TxBlock,
SenderShardID: 0,
ReceiverShardID: 0,
TxHashes: [][]byte{txHash},
},
},
}

address1 := "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u"
address2 := "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht"
initialTx := &transaction.Transaction{
Nonce: 1000,
SndAddr: decodeAddress(address1),
RcvAddr: decodeAddress(address2),
GasLimit: 15406000,
GasPrice: 1000000000,
Value: big.NewInt(0),
InnerTransactions: []*transaction.Transaction{
{
Nonce: 10,
SndAddr: decodeAddress(address1),
RcvAddr: decodeAddress(address2),
GasLimit: 15406000,
GasPrice: 1000000000,
Value: big.NewInt(0),
},
{
Nonce: 20,
SndAddr: decodeAddress(address1),
RcvAddr: decodeAddress(address2),
GasLimit: 15406000,
GasPrice: 1000000000,
Value: big.NewInt(1000),
},
},
}

txInfo := &outport.TxInfo{
Transaction: initialTx,
FeeInfo: &outport.FeeInfo{
GasUsed: 10556000,
Fee: big.NewInt(2257820000000000),
InitialPaidFee: big.NewInt(2306320000000000),
},
ExecutionOrder: 0,
}

pool := &outport.TransactionPool{
Transactions: map[string]*outport.TxInfo{
hex.EncodeToString(txHash): txInfo,
},
}
err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards))
require.Nil(t, err)

ids := []string{hex.EncodeToString(txHash)}
genericResponse := &GenericResponse{}
err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse)
require.Nil(t, err)

require.JSONEq(t,
readExpectedResult("./testdata/relayedTxV3/relayed-tx-v3.json"),
string(genericResponse.Docs[0].Source),
)
}
54 changes: 54 additions & 0 deletions integrationtests/testdata/relayedTxV3/relayed-tx-v3.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
"miniBlockHash": "b3222be4506429fa237393c56b732a283eea5ccab465deb66b6348083793b6c9",
"nonce": 1000,
"round": 50,
"value": "0",
"valueNum": 0,
"receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
"sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u",
"receiverShard": 0,
"senderShard": 0,
"gasPrice": 1000000000,
"gasLimit": 15406000,
"gasUsed": 10556000,
"fee": "2257820000000000",
"feeNum": 0.00225782,
"initialPaidFee": "2306320000000000",
"data": null,
"signature": "",
"timestamp": 5040,
"status": "success",
"searchOrder": 0,
"receivers": [
"erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
"erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht"
],
"receiversShardIDs": [
1,
1
],
"operation": "transfer",
"isRelayed": true,
"innerTransactions": [
{
"nonce": 10,
"value": "0",
"receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
"sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u",
"gasPrice": 1000000000,
"gasLimit": 15406000,
"chainID": "",
"version": 0
},
{
"nonce": 20,
"value": "1000",
"receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
"sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u",
"gasPrice": 1000000000,
"gasLimit": 15406000,
"chainID": "",
"version": 0
}
]
}
5 changes: 5 additions & 0 deletions mock/databaseWriterStub.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ type DatabaseWriterStub struct {
DoScrollRequestCalled func(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error) error
}

// PutMappings -
func (dwm *DatabaseWriterStub) PutMappings(_ string, _ *bytes.Buffer) error {
return nil
}

// UpdateByQuery -
func (dwm *DatabaseWriterStub) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error {
return nil
Expand Down
17 changes: 15 additions & 2 deletions process/elasticproc/elasticProcessor.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters"
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags"
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo"
"github.com/multiversx/mx-chain-es-indexer-go/templates"
logger "github.com/multiversx/mx-chain-logger-go"
)

Expand All @@ -44,6 +45,7 @@ type ArgElasticProcessor struct {
ImportDB bool
IndexTemplates map[string]*bytes.Buffer
IndexPolicies map[string]*bytes.Buffer
ExtraMappings []templates.ExtraMapping
EnabledIndexes map[string]struct{}
TransactionsProc DBTransactionsHandler
AccountsProc DBAccountHandler
Expand Down Expand Up @@ -94,7 +96,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err
bulkRequestMaxSize: arguments.BulkRequestMaxSize,
}

err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies)
err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings)
if err != nil {
return nil, err
}
Expand All @@ -105,7 +107,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err
}

// TODO move all the index create part in a new component
func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*bytes.Buffer) error {
func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*bytes.Buffer, extraMappings []templates.ExtraMapping) error {
err := ei.createOpenDistroTemplates(indexTemplates)
if err != nil {
return err
Expand Down Expand Up @@ -135,6 +137,17 @@ func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*b
return err
}

return ei.addExtraMappings(extraMappings)
}

func (ei *elasticProcessor) addExtraMappings(extraMappings []templates.ExtraMapping) error {
for _, mappingsTuple := range extraMappings {
err := ei.elasticClient.PutMappings(mappingsTuple.Index, mappingsTuple.Mappings)
if err != nil {
return err
}
}

return nil
}

Expand Down
5 changes: 5 additions & 0 deletions process/elasticproc/factory/elasticProcessorFactory.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E
if err != nil {
return nil, err
}
extraMappings, err := templatesAndPoliciesReader.GetExtraMappings()
if err != nil {
return nil, err
}

enabledIndexesMap := make(map[string]struct{})
for _, index := range arguments.EnabledIndexes {
Expand Down Expand Up @@ -119,6 +123,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E
UseKibana: arguments.UseKibana,
IndexTemplates: indexTemplates,
IndexPolicies: indexPolicies,
ExtraMappings: extraMappings,
OperationsProc: operationsProc,
ImportDB: arguments.ImportDB,
Version: arguments.Version,
Expand Down
1 change: 1 addition & 0 deletions process/elasticproc/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ type DatabaseClientHandler interface {
DoCountRequest(ctx context.Context, index string, body []byte) (uint64, error)
UpdateByQuery(ctx context.Context, index string, buff *bytes.Buffer) error

PutMappings(indexName string, mappings *bytes.Buffer) error
CheckAndCreateIndex(index string) error
CheckAndCreateAlias(alias string, index string) error
CheckAndCreateTemplate(templateName string, template *bytes.Buffer) error
Expand Down
7 changes: 6 additions & 1 deletion process/elasticproc/templatesAndPolicies/interface.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
package templatesAndPolicies

import "bytes"
import (
"bytes"

"github.com/multiversx/mx-chain-es-indexer-go/templates"
)

// TemplatesAndPoliciesHandler defines the actions that a templates and policies handler should do
type TemplatesAndPoliciesHandler interface {
GetElasticTemplatesAndPolicies() (map[string]*bytes.Buffer, map[string]*bytes.Buffer, error)
GetExtraMappings() ([]templates.ExtraMapping, error)
}
Loading

0 comments on commit 51bea3e

Please sign in to comment.