From 2920f61251d090e5fc092ea19abe7506f8afae7d Mon Sep 17 00:00:00 2001 From: Ran Mishael Date: Wed, 25 Jan 2023 12:40:00 +0100 Subject: [PATCH] PRT-361 hotfix changes to provider sequence number error. --- relayer/sentry/sentry.go | 29 ++++++++++++++- relayer/server.go | 80 +++++++++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/relayer/sentry/sentry.go b/relayer/sentry/sentry.go index c9b8257f3b..3b1839946a 100755 --- a/relayer/sentry/sentry.go +++ b/relayer/sentry/sentry.go @@ -6,6 +6,8 @@ import ( "encoding/binary" "encoding/json" "fmt" + "math" + "math/rand" "regexp" "sort" "strconv" @@ -100,6 +102,13 @@ type ProviderHashesConsensus struct { agreeingProviders map[string]providerDataContainer } +type RewardHandler struct { + epochEventTriggered bool + delayRewardBy int + waitedBlocks int + blockHeight int64 +} + type Sentry struct { ClientCtx client.Context rpcClient rpcclient.Client @@ -595,13 +604,17 @@ func (s *Sentry) PrintExpectedPayments() string { } func (s *Sentry) Start(ctx context.Context) { + rewardHandler := &RewardHandler{} + if !s.isUser { // listen for transactions for proof of relay payment go s.ListenForTXEvents(ctx) } + // // Listen for blockchain events for e := range s.NewBlockEvents { + // switch data := e.Data.(type) { case tenderminttypes.EventDataNewBlock: // @@ -620,7 +633,11 @@ func (s *Sentry) Start(ctx context.Context) { } if s.newEpochCb != nil { - go s.newEpochCb(data.Block.Height - StaleEpochDistance*int64(s.GetEpochSize())) // Currently this is only askForRewards + epochSize := s.GetEpochSize() + rewardHandler.epochEventTriggered = true + rewardHandler.delayRewardBy = int(math.Abs(rand.Float64() * float64(epochSize/2))) + rewardHandler.blockHeight = (data.Block.Height - StaleEpochDistance*int64(epochSize)) + utils.LavaFormatInfo("delaying to ask rewards", &map[string]string{"delayedBlocks": strconv.Itoa(rewardHandler.delayRewardBy)}) } // @@ -702,6 +719,16 @@ func (s *Sentry) Start(ctx context.Context) { { } } + + // This happens at the end of the epoch switch case as we might get 0 delay and want to trigger the reward in the same block + if s.newEpochCb != nil && rewardHandler.epochEventTriggered { + if rewardHandler.waitedBlocks >= rewardHandler.delayRewardBy { + utils.LavaFormatInfo("Asking for rewards", &map[string]string{"delayedBlocks": strconv.Itoa(rewardHandler.delayRewardBy)}) + go s.newEpochCb(rewardHandler.blockHeight) // Currently this is only askForRewards + rewardHandler = &RewardHandler{} // reset reward handler to default. + } + rewardHandler.waitedBlocks += 1 + } } } diff --git a/relayer/server.go b/relayer/server.go index b099ab60b9..6a2b3533ee 100644 --- a/relayer/server.go +++ b/relayer/server.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "os/signal" + "regexp" "strconv" "strings" "sync" @@ -48,7 +49,7 @@ import ( ) const ( - RETRY_INCORRECT_SEQUENCE = 3 + RETRY_INCORRECT_SEQUENCE = 5 TimeWaitInitializeChainSentry = 10 RetryInitAttempts = 10 ) @@ -255,15 +256,23 @@ func askForRewards(staleEpochHeight int64) { hasSequenceError := false success := false idx := -1 + sequenceNumberParsed := 0 summarizedTransactionResult := "" for ; idx < RETRY_INCORRECT_SEQUENCE && !success; idx++ { msg := pairingtypes.NewMsgRelayPayment(g_sentry.Acc, relays, strconv.FormatUint(g_serverID, 10)) g_sentry.ClientCtx.Output = &myWriter if hasSequenceError { // a retry - _, seq, err := g_sentry.ClientCtx.AccountRetriever.GetAccountNumberSequence(g_sentry.ClientCtx, g_sentry.ClientCtx.GetFromAddress()) - if err != nil { - utils.LavaFormatError("failed to get correct sequence number for account, give up", err, nil) - break // give up + var seq uint64 + if sequenceNumberParsed != 0 { + utils.LavaFormatInfo("Sequence Number extracted from transaction error, retrying", &map[string]string{"sequence": strconv.Itoa(sequenceNumberParsed)}) + seq = uint64(sequenceNumberParsed) + } else { + var err error + _, seq, err = g_sentry.ClientCtx.AccountRetriever.GetAccountNumberSequence(g_sentry.ClientCtx, g_sentry.ClientCtx.GetFromAddress()) + if err != nil { + utils.LavaFormatError("failed to get correct sequence number for account, give up", err, nil) + break // give up + } } g_txFactory = g_txFactory.WithSequence(seq) myWriter.Reset() @@ -271,38 +280,30 @@ func askForRewards(staleEpochHeight int64) { "SeqNum": strconv.FormatUint(seq, 10), }) } + + var transactionResult string err := sentry.CheckProfitabilityAndBroadCastTx(g_sentry.ClientCtx, g_txFactory, msg) if err != nil { - utils.LavaFormatError("Sending CheckProfitabilityAndBroadCastTx failed", err, &map[string]string{ + utils.LavaFormatWarning("Sending CheckProfitabilityAndBroadCastTx failed", err, &map[string]string{ "msg": fmt.Sprintf("%+v", msg), }) - } - - transactionResult := myWriter.String() - summarized, transactionResults := summarizeTransactionResult(transactionResult) - summarizedTransactionResult = summarized - - var returnCode uint64 - splitted := strings.Split(transactionResults[0], ":") - if len(splitted) < 2 { - utils.LavaFormatError("Failed to parse transaction result", err, &map[string]string{ - "parsing data": transactionResult, - }) - returnCode = 1 // just not zero + transactionResult = err.Error() // incase we got an error the tx result is basically the error } else { - returnCode, err = strconv.ParseUint(splitted[1], 10, 32) - if err != nil { - utils.LavaFormatError("Failed to parse transaction result", err, &map[string]string{ - "parsing data": transactionResult, - }) - returnCode = 1 // just not zero - } + transactionResult = myWriter.String() } + + var returnCode int + summarizedTransactionResult, returnCode = parseTransactionResult(transactionResult) - if returnCode == 0 { // if we get some other error which isnt then keep retrying + if returnCode == 0 { // if we get some other code which isn't 0 then keep retrying success = true - } else if strings.Contains(summarized, "incorrect account sequence") { + } else if strings.Contains(transactionResult, "account sequence") { hasSequenceError = true + sequenceNumberParsed, err = findSequenceNumber(transactionResult) + if err != nil { + utils.LavaFormatWarning("Failed findSequenceNumber", err, &map[string]string{"sequence": transactionResult}) + } + summarizedTransactionResult = transactionResult } } @@ -313,7 +314,17 @@ func askForRewards(staleEpochHeight int64) { } } -func summarizeTransactionResult(transactionResult string) (string, []string) { +// extract requested sequence number from tx error. +func findSequenceNumber(sequence string) (int, error) { + re := regexp.MustCompile(`expected (\d+), got (\d+)`) + match := re.FindStringSubmatch(sequence) + if match == nil || len(match) < 2 { + return 0, utils.LavaFormatWarning("Failed to parse sequence number from error", nil, &map[string]string{"sequence": sequence}) + } + return strconv.Atoi(match[1]) // atoi return 0 upon error, so it will be ok when sequenceNumberParsed uses it +} + +func parseTransactionResult(transactionResult string) (string, int) { transactionResult = strings.ReplaceAll(transactionResult, ": ", ":") transactionResults := strings.Split(transactionResult, "\n") summarizedResult := "" @@ -322,7 +333,16 @@ func summarizeTransactionResult(transactionResult string) (string, []string) { summarizedResult = summarizedResult + str + ", " } } - return summarizedResult, transactionResults + re := regexp.MustCompile(`code:(\d+)`) // extracting code from transaction result (in format code:%d) + match := re.FindStringSubmatch(transactionResult) + if match == nil || len(match) < 2 { + return summarizedResult, 1 // not zero + } + retCode, err := strconv.Atoi(match[1]) // extract return code. + if err != nil { + return summarizedResult, 1 // not zero + } + return summarizedResult, retCode } func getRelayUser(in *pairingtypes.RelayRequest) (tenderbytes.HexBytes, error) {