fix: adjust evm outbound tracker reporter to avoid submitting invalid…

… hashes (#2628)

* refactor and fix evm outbound tracker reporter to avoid invalid hashes; print log when outbound tracker is full of invalid hashes

* add changelog entry

* used predefined log fields

* remove repeated fields information from log message; Devops team would configure Datadog to show the fields

* remove redundant fields in log message; unified logs

* remove pending transaction map from observer; the outbound tracker reporter will no longer report pending hash

* use bg.Work() to launch outbound tracker reporter goroutines

* bring the checking EnsureNoTrackers() back

* add more rationale to EVM outbound tracker submission

* sync observer and signers without wait on startup

* try fixing tss migration E2E failure by increase timeout

changelog.md

@@ -7,6 +7,7 @@
 * [2568](https://github.com/zeta-chain/node/pull/2568) - improve AppContext by converging chains, chainParams, enabledChains, and additionalChains into a single zctx.Chain
 * [2597](https://github.com/zeta-chain/node/pull/2597) - Add generic rpc metrics to zetaclient
 * [2634](https://github.com/zeta-chain/node/pull/2634) - add support for EIP-1559 gas fees
+* [2628](https://github.com/zeta-chain/node/pull/2628) - avoid submitting invalid hashes to outbound tracker
 
 
 ## v19.0.1

e2e/utils/zetacore.go

@@ -20,7 +20,7 @@ const (
 	AdminPolicyName       = "admin"
 	OperationalPolicyName = "operational"
 
-	DefaultCctxTimeout = 4 * time.Minute
+	DefaultCctxTimeout = 6 * time.Minute
 )
 
 // WaitCctxMinedByInboundHash waits until cctx is mined; returns the cctxIndex (the last one)

pkg/constant/constant.go

@@ -1,6 +1,12 @@
 package constant
 
+import "time"
+
 const (
+	// ZetaBlockTime is the block time of the ZetaChain network
+	// It's a rough estimate that can be used in non-critical path to estimate the time of a block
+	ZetaBlockTime = 6000 * time.Millisecond
+
 	// DonationMessage is the message for donation transactions
 	// Transaction sent to the TSS or ERC20 Custody address containing this message are considered as a donation
 	DonationMessage = "I am rich!"

zetaclient/chains/base/observer.go

@@ -17,6 +17,7 @@ import (
 	observertypes "github.com/zeta-chain/zetacore/x/observer/types"
 	"github.com/zeta-chain/zetacore/zetaclient/chains/interfaces"
 	"github.com/zeta-chain/zetacore/zetaclient/db"
+	"github.com/zeta-chain/zetacore/zetaclient/logs"
 	"github.com/zeta-chain/zetacore/zetaclient/metrics"
 	clienttypes "github.com/zeta-chain/zetacore/zetaclient/types"
 	"github.com/zeta-chain/zetacore/zetaclient/zetacore"
@@ -295,13 +296,13 @@ func (ob *Observer) Logger() *ObserverLogger {
 
 // WithLogger attaches a new logger to the observer.
 func (ob *Observer) WithLogger(logger Logger) *Observer {
-	chainLogger := logger.Std.With().Int64("chain", ob.chain.ChainId).Logger()
+	chainLogger := logger.Std.With().Int64(logs.FieldChain, ob.chain.ChainId).Logger()
 	ob.logger = ObserverLogger{
 		Chain:      chainLogger,
-		Inbound:    chainLogger.With().Str("module", "inbound").Logger(),
-		Outbound:   chainLogger.With().Str("module", "outbound").Logger(),
-		GasPrice:   chainLogger.With().Str("module", "gasprice").Logger(),
-		Headers:    chainLogger.With().Str("module", "headers").Logger(),
+		Inbound:    chainLogger.With().Str(logs.FieldModule, logs.ModNameInbound).Logger(),
+		Outbound:   chainLogger.With().Str(logs.FieldModule, logs.ModNameOutbound).Logger(),
+		GasPrice:   chainLogger.With().Str(logs.FieldModule, logs.ModNameGasPrice).Logger(),
+		Headers:    chainLogger.With().Str(logs.FieldModule, logs.ModNameHeaders).Logger(),
 		Compliance: logger.Compliance,
 	}
 	return ob

zetaclient/chains/base/signer.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/zeta-chain/zetacore/pkg/chains"
 	"github.com/zeta-chain/zetacore/zetaclient/chains/interfaces"
+	"github.com/zeta-chain/zetacore/zetaclient/logs"
 	"github.com/zeta-chain/zetacore/zetaclient/metrics"
 )
 
@@ -38,7 +39,10 @@ func NewSigner(chain chains.Chain, tss interfaces.TSSSigner, ts *metrics.Telemet
 		tss:   tss,
 		ts:    ts,
 		logger: Logger{
-			Std:        logger.Std.With().Int64("chain", chain.ChainId).Str("module", "signer").Logger(),
+			Std: logger.Std.With().
+				Int64(logs.FieldChain, chain.ChainId).
+				Str(logs.FieldModule, "signer").
+				Logger(),
 			Compliance: logger.Compliance,
 		},
 		outboundBeingReported: make(map[string]bool),

zetaclient/chains/evm/constant.go

@@ -3,12 +3,13 @@ package evm
 import "time"
 
 const (
-	// ZetaBlockTime is the block time of the Zeta network
-	ZetaBlockTime = 6500 * time.Millisecond
-
 	// OutboundInclusionTimeout is the timeout for waiting for an outbound to be included in a block
 	OutboundInclusionTimeout = 20 * time.Minute
 
+	// ReorgProtectBlockCount is confirmations count to protect against reorg
+	// Short 1~2 block reorgs could happen often on Ethereum due to network congestion or block production race conditions
+	ReorgProtectBlockCount = 2
+
 	// OutboundTrackerReportTimeout is the timeout for waiting for an outbound tracker report
 	OutboundTrackerReportTimeout = 10 * time.Minute
 

zetaclient/chains/evm/observer/observer.go

@@ -43,9 +43,6 @@ type Observer struct {
 	// evmJSONRPC is the EVM JSON RPC client for the observed chain
 	evmJSONRPC interfaces.EVMJSONRPCClient
 
-	// outboundPendingTransactions is the map to index pending transactions by hash
-	outboundPendingTransactions map[string]*ethtypes.Transaction
-
 	// outboundConfirmedReceipts is the map to index confirmed receipts by hash
 	outboundConfirmedReceipts map[string]*ethtypes.Receipt
 
@@ -92,7 +89,6 @@ func NewObserver(
 		Observer:                      *baseObserver,
 		evmClient:                     evmClient,
 		evmJSONRPC:                    ethrpc.NewEthRPC(evmCfg.Endpoint),
-		outboundPendingTransactions:   make(map[string]*ethtypes.Transaction),
 		outboundConfirmedReceipts:     make(map[string]*ethtypes.Receipt),
 		outboundConfirmedTransactions: make(map[string]*ethtypes.Transaction),
 		priorityFeeConfig:             priorityFeeConfig{},
@@ -230,25 +226,10 @@ func (ob *Observer) WatchRPCStatus(ctx context.Context) error {
 	}
 }
 
-// SetPendingTx sets the pending transaction in memory
-func (ob *Observer) SetPendingTx(nonce uint64, transaction *ethtypes.Transaction) {
-	ob.Mu().Lock()
-	defer ob.Mu().Unlock()
-	ob.outboundPendingTransactions[ob.OutboundID(nonce)] = transaction
-}
-
-// GetPendingTx gets the pending transaction from memory
-func (ob *Observer) GetPendingTx(nonce uint64) *ethtypes.Transaction {
-	ob.Mu().Lock()
-	defer ob.Mu().Unlock()
-	return ob.outboundPendingTransactions[ob.OutboundID(nonce)]
-}
-
 // SetTxNReceipt sets the receipt and transaction in memory
 func (ob *Observer) SetTxNReceipt(nonce uint64, receipt *ethtypes.Receipt, transaction *ethtypes.Transaction) {
 	ob.Mu().Lock()
 	defer ob.Mu().Unlock()
-	delete(ob.outboundPendingTransactions, ob.OutboundID(nonce)) // remove pending transaction, if any
 	ob.outboundConfirmedReceipts[ob.OutboundID(nonce)] = receipt
 	ob.outboundConfirmedTransactions[ob.OutboundID(nonce)] = transaction
 }