From 041df59fe4d012ecd293a0bb530e1c25dae9a381 Mon Sep 17 00:00:00 2001 From: Dmitry S <11892559+swift1337@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:39:04 +0100 Subject: [PATCH] feat(ton): adjacent TON tasks (#3075) * Implement InboundTracker; minor refactoring * E2E: Concurrent withdrawals [WIP] * E2E: Concurrent withdrawals * Improve signer broadcasting & logging * Add e2e for deposit & refund * Update changelog * Address PR comments * Address PR comments [2] --- changelog.md | 3 + cmd/zetae2e/local/local.go | 2 + e2e/e2etests/e2etests.go | 22 +++- e2e/e2etests/test_ton_deposit_refund.go | 50 ++++++++ e2e/e2etests/test_ton_withdrawal.go | 7 +- .../test_ton_withdrawal_concurrent.go | 74 +++++++++++ e2e/runner/setup_ton.go | 35 +++-- e2e/runner/ton.go | 47 ++++++- e2e/runner/ton/deployer.go | 11 +- e2e/runner/zeta.go | 9 +- pkg/contracts/ton/gateway_msg.go | 9 ++ zetaclient/chains/ton/observer/inbound.go | 103 ++++++++++++--- .../chains/ton/observer/inbound_test.go | 120 +++++++++++++----- zetaclient/chains/ton/observer/observer.go | 19 ++- .../chains/ton/observer/observer_test.go | 21 +++ zetaclient/chains/ton/signer/signer.go | 93 +++++++++++--- zetaclient/chains/ton/signer/signer_test.go | 25 ++++ zetaclient/orchestrator/orchestrator.go | 7 +- 18 files changed, 546 insertions(+), 111 deletions(-) create mode 100644 e2e/e2etests/test_ton_deposit_refund.go create mode 100644 e2e/e2etests/test_ton_withdrawal_concurrent.go diff --git a/changelog.md b/changelog.md index 4ba4b6fcf7..3c2cd59ccb 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,9 @@ * [2984](https://github.com/zeta-chain/node/pull/2984) - add Whitelist message ability to whitelist SPL tokens on Solana +### Tests +* [3075](https://github.com/zeta-chain/node/pull/3075) - ton: withdraw concurrent, deposit & revert. + ## v21.0.0 ### Features diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 986e9e4689..d5b4ac8076 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -426,7 +426,9 @@ func localE2ETest(cmd *cobra.Command, _ []string) { tonTests := []string{ e2etests.TestTONDepositName, e2etests.TestTONDepositAndCallName, + e2etests.TestTONDepositAndCallRefundName, e2etests.TestTONWithdrawName, + e2etests.TestTONWithdrawConcurrentName, } eg.Go(tonTestRoutine(conf, deployerRunner, verbose, tonTests...)) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 9f86120bb0..82764cb537 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -65,9 +65,11 @@ const ( /** * TON tests */ - TestTONDepositName = "ton_deposit" - TestTONDepositAndCallName = "ton_deposit_and_call" - TestTONWithdrawName = "ton_withdraw" + TestTONDepositName = "ton_deposit" + TestTONDepositAndCallName = "ton_deposit_and_call" + TestTONDepositAndCallRefundName = "ton_deposit_refund" + TestTONWithdrawName = "ton_withdraw" + TestTONWithdrawConcurrentName = "ton_withdraw_concurrent" /* Bitcoin tests @@ -479,6 +481,14 @@ var AllE2ETests = []runner.E2ETest{ }, TestTONDepositAndCall, ), + runner.NewE2ETest( + TestTONDepositAndCallRefundName, + "deposit TON into ZEVM and call a smart contract that reverts; expect refund", + []runner.ArgDefinition{ + {Description: "amount in nano tons", DefaultValue: "1000000000"}, // 1.0 TON + }, + TestTONDepositAndCallRefund, + ), runner.NewE2ETest( TestTONWithdrawName, "withdraw TON from ZEVM", @@ -487,6 +497,12 @@ var AllE2ETests = []runner.E2ETest{ }, TestTONWithdraw, ), + runner.NewE2ETest( + TestTONWithdrawConcurrentName, + "withdraw TON from ZEVM for several recipients simultaneously", + []runner.ArgDefinition{}, + TestTONWithdrawConcurrent, + ), /* Bitcoin tests */ diff --git a/e2e/e2etests/test_ton_deposit_refund.go b/e2e/e2etests/test_ton_deposit_refund.go new file mode 100644 index 0000000000..3259e1d1d6 --- /dev/null +++ b/e2e/e2etests/test_ton_deposit_refund.go @@ -0,0 +1,50 @@ +package e2etests + +import ( + "github.com/stretchr/testify/require" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + testcontract "github.com/zeta-chain/node/testutil/contracts" + cctypes "github.com/zeta-chain/node/x/crosschain/types" +) + +func TestTONDepositAndCallRefund(r *runner.E2ERunner, args []string) { + require.Len(r, args, 1) + + // Given amount and arbitrary call data + var ( + amount = parseUint(r, args[0]) + data = []byte("hello reverter") + ) + + // Given deployer mock revert contract + // deploy a reverter contract in ZEVM + reverterAddr, _, _, err := testcontract.DeployReverter(r.ZEVMAuth, r.ZEVMClient) + require.NoError(r, err) + r.Logger.Info("Reverter contract deployed at: %s", reverterAddr.String()) + + // ACT + // Send a deposit and call transaction from the deployer (faucet) + // to the reverter contract + cctx, err := r.TONDepositAndCall( + &r.TONDeployer.Wallet, + amount, + reverterAddr, + data, + runner.TONExpectStatus(cctypes.CctxStatus_Reverted), + ) + + // ASSERT + require.NoError(r, err) + r.Logger.CCTX(*cctx, "ton_deposit_and_refund") + + // Check the error carries the revert executed. + // tolerate the error in both the ErrorMessage field and the StatusMessage field + if cctx.CctxStatus.ErrorMessage != "" { + require.Contains(r, cctx.CctxStatus.ErrorMessage, "revert executed") + return + } + + require.Contains(r, cctx.CctxStatus.StatusMessage, utils.ErrHashRevertFoo) +} diff --git a/e2e/e2etests/test_ton_withdrawal.go b/e2e/e2etests/test_ton_withdrawal.go index d78f624716..db75fa4d28 100644 --- a/e2e/e2etests/test_ton_withdrawal.go +++ b/e2e/e2etests/test_ton_withdrawal.go @@ -12,9 +12,6 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/ton/liteapi" ) -// TODO: Add "withdraw_many_concurrent" test -// https://github.com/zeta-chain/node/issues/3044 - func TestTONWithdraw(r *runner.E2ERunner, args []string) { // ARRANGE require.Len(r, args, 1) @@ -34,7 +31,7 @@ func TestTONWithdraw(r *runner.E2ERunner, args []string) { tonRecipient, err := deployer.CreateWallet(r.Ctx, toncontracts.Coins(1)) require.NoError(r, err) - tonRecipientBalanceBefore, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress()) + tonRecipientBalanceBefore, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress(), true) require.NoError(r, err) r.Logger.Info("Recipient's TON balance before withdrawal: %s", toncontracts.FormatCoins(tonRecipientBalanceBefore)) @@ -61,7 +58,7 @@ func TestTONWithdraw(r *runner.E2ERunner, args []string) { ) // Make sure that recipient's TON balance has increased - tonRecipientBalanceAfter, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress()) + tonRecipientBalanceAfter, err := deployer.GetBalanceOf(r.Ctx, tonRecipient.GetAddress(), true) require.NoError(r, err) r.Logger.Info("Recipient's balance after withdrawal: %s", toncontracts.FormatCoins(tonRecipientBalanceAfter)) diff --git a/e2e/e2etests/test_ton_withdrawal_concurrent.go b/e2e/e2etests/test_ton_withdrawal_concurrent.go new file mode 100644 index 0000000000..fb4217013d --- /dev/null +++ b/e2e/e2etests/test_ton_withdrawal_concurrent.go @@ -0,0 +1,74 @@ +package e2etests + +import ( + "math/rand" + "sync" + + "cosmossdk.io/math" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/require" + "github.com/tonkeeper/tongo/ton" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" + "github.com/zeta-chain/node/testutil/sample" + cc "github.com/zeta-chain/node/x/crosschain/types" +) + +// TestTONWithdrawConcurrent makes sure that multiple concurrent +// withdrawals will be eventually processed by sequentially increasing Gateway nonce +// and that zetaclient tolerates "invalid nonce" error from RPC. +func TestTONWithdrawConcurrent(r *runner.E2ERunner, _ []string) { + // ARRANGE + // Given a deployer + _, deployer := r.Ctx, r.TONDeployer + + const recipientsCount = 10 + + // Fire withdrawals. Note that zevm sender is r.ZEVMAuth + var wg sync.WaitGroup + for i := 0; i < recipientsCount; i++ { + // ARRANGE + // Given multiple recipients WITHOUT deployed wallet-contracts + // and withdrawal amounts between 1 and 5 TON + var ( + // #nosec G404: it's a test + amountCoins = 1 + rand.Intn(5) + // #nosec G115 test - always in range + amount = toncontracts.Coins(uint64(amountCoins)) + recipient = sample.GenerateTONAccountID() + ) + + // ACT + r.Logger.Info( + "Withdrawal #%d: sending %s to %s", + i+1, + toncontracts.FormatCoins(amount), + recipient.ToRaw(), + ) + + approvedAmount := amount.Add(toncontracts.Coins(1)) + tx := r.SendWithdrawTONZRC20(recipient, amount.BigInt(), approvedAmount.BigInt()) + + wg.Add(1) + + go func(number int, recipient ton.AccountID, amount math.Uint, tx *ethtypes.Transaction) { + defer wg.Done() + + // wait for the cctx to be mined + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.CctxTimeout) + + // ASSERT + utils.RequireCCTXStatus(r, cctx, cc.CctxStatus_OutboundMined) + r.Logger.Info("Withdrawal #%d complete! cctx index: %s", number, cctx.Index) + + // Check recipient's balance ON TON + balance, err := deployer.GetBalanceOf(r.Ctx, recipient, false) + require.NoError(r, err, "failed to get balance of %s", recipient.ToRaw()) + require.Equal(r, amount.Uint64(), balance.Uint64()) + }(i+1, recipient, amount, tx) + } + + wg.Wait() +} diff --git a/e2e/runner/setup_ton.go b/e2e/runner/setup_ton.go index 49feb5d95a..10954b5f43 100644 --- a/e2e/runner/setup_ton.go +++ b/e2e/runner/setup_ton.go @@ -11,6 +11,7 @@ import ( "github.com/zeta-chain/node/pkg/chains" "github.com/zeta-chain/node/pkg/constant" toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" + cctxtypes "github.com/zeta-chain/node/x/crosschain/types" observertypes "github.com/zeta-chain/node/x/observer/types" ) @@ -54,29 +55,35 @@ func (r *E2ERunner) SetupTON() error { ) // 3. Check that the gateway indeed was deployed and has desired TON balance. - gwBalance, err := deployer.GetBalanceOf(ctx, gwAccount.ID) - if err != nil { + gwBalance, err := deployer.GetBalanceOf(ctx, gwAccount.ID, true) + switch { + case err != nil: return errors.Wrap(err, "unable to get balance of TON gateway") + case gwBalance.IsZero(): + return fmt.Errorf("TON gateway balance is zero") } - if gwBalance.IsZero() { - return fmt.Errorf("TON gateway balance is zero") + // 4. Set chain params & chain nonce + if err := r.ensureTONChainParams(gwAccount); err != nil { + return errors.Wrap(err, "unable to ensure TON chain params") } - // 4. Deposit 100 TON deployer to Zevm Auth - gw := toncontracts.NewGateway(gwAccount.ID) - veryFirstDeposit := toncontracts.Coins(1000) + r.TONDeployer = deployer + r.TONGateway = toncontracts.NewGateway(gwAccount.ID) + + // 5. Deposit 10000 TON deployer to Zevm Auth + veryFirstDeposit := toncontracts.Coins(10000) zevmRecipient := r.ZEVMAuth.From - err = gw.SendDeposit(ctx, &deployer.Wallet, veryFirstDeposit, zevmRecipient, 0) - if err != nil { - return errors.Wrap(err, "unable to send deposit to TON gateway") + gwDeposit, err := r.TONDeposit(&deployer.Wallet, veryFirstDeposit, zevmRecipient) + switch { + case err != nil: + return errors.Wrap(err, "unable to deposit TON to Zevm Auth") + case gwDeposit.CctxStatus.Status != cctxtypes.CctxStatus_OutboundMined: + return errors.New("gateway deposit CCTX is not mined") } - r.TONDeployer = deployer - r.TONGateway = gw - - return r.ensureTONChainParams(gwAccount) + return nil } func (r *E2ERunner) ensureTONChainParams(gw *ton.AccountInit) error { diff --git a/e2e/runner/ton.go b/e2e/runner/ton.go index eb8e5346ac..369bfa0df3 100644 --- a/e2e/runner/ton.go +++ b/e2e/runner/ton.go @@ -1,11 +1,13 @@ package runner import ( + "encoding/hex" "math/big" "time" "cosmossdk.io/math" eth "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/pkg/errors" "github.com/stretchr/testify/require" "github.com/tonkeeper/tongo/ton" @@ -23,6 +25,20 @@ import ( // https://docs.ton.org/develop/smart-contracts/guidelines/message-modes-cookbook const tonDepositSendCode = toncontracts.SendFlagSeparateFees + toncontracts.SendFlagIgnoreErrors +// currently implemented only for DepositAndCall, +// can be adopted for all TON ops +type tonOpts struct { + expectedStatus cctypes.CctxStatus +} + +type TONOpt func(t *tonOpts) + +func TONExpectStatus(status cctypes.CctxStatus) TONOpt { + return func(t *tonOpts) { + t.expectedStatus = status + } +} + // TONDeposit deposit TON to Gateway contract func (r *E2ERunner) TONDeposit( sender *wallet.Wallet, @@ -56,7 +72,7 @@ func (r *E2ERunner) TONDeposit( } // Wait for cctx - cctx := r.WaitForSpecificCCTX(filter, time.Minute) + cctx := r.WaitForSpecificCCTX(filter, cctypes.CctxStatus_OutboundMined, time.Minute) return cctx, nil } @@ -67,7 +83,13 @@ func (r *E2ERunner) TONDepositAndCall( amount math.Uint, zevmRecipient eth.Address, callData []byte, + opts ...TONOpt, ) (*cctypes.CrossChainTx, error) { + cfg := &tonOpts{expectedStatus: cctypes.CctxStatus_OutboundMined} + for _, opt := range opts { + opt(cfg) + } + chain := chains.TONLocalnet require.NotNil(r, r.TONGateway, "TON Gateway is not initialized") @@ -91,18 +113,26 @@ func (r *E2ERunner) TONDepositAndCall( } filter := func(cctx *cctypes.CrossChainTx) bool { + memo := zevmRecipient.Bytes() + memo = append(memo, callData...) + return cctx.InboundParams.SenderChainId == chain.ChainId && - cctx.InboundParams.Sender == sender.GetAddress().ToRaw() + cctx.InboundParams.Sender == sender.GetAddress().ToRaw() && + cctx.RelayedMessage == hex.EncodeToString(memo) } // Wait for cctx - cctx := r.WaitForSpecificCCTX(filter, time.Minute) + cctx := r.WaitForSpecificCCTX(filter, cfg.expectedStatus, time.Minute) return cctx, nil } -// WithdrawTONZRC20 withdraws an amount of ZRC20 TON tokens -func (r *E2ERunner) WithdrawTONZRC20(to ton.AccountID, amount *big.Int, approveAmount *big.Int) *cctypes.CrossChainTx { +// SendWithdrawTONZRC20 sends withdraw tx of TON ZRC20 tokens +func (r *E2ERunner) SendWithdrawTONZRC20( + to ton.AccountID, + amount *big.Int, + approveAmount *big.Int, +) *ethtypes.Transaction { // approve tx, err := r.TONZRC20.Approve(r.ZEVMAuth, r.TONZRC20Addr, approveAmount) require.NoError(r, err) @@ -119,6 +149,13 @@ func (r *E2ERunner) WithdrawTONZRC20(to ton.AccountID, amount *big.Int, approveA utils.RequireTxSuccessful(r, receipt, "withdraw") r.Logger.Info("Receipt txhash %s status %d", receipt.TxHash, receipt.Status) + return tx +} + +// WithdrawTONZRC20 withdraws an amount of ZRC20 TON tokens and waits for the cctx to be mined +func (r *E2ERunner) WithdrawTONZRC20(to ton.AccountID, amount *big.Int, approveAmount *big.Int) *cctypes.CrossChainTx { + tx := r.SendWithdrawTONZRC20(to, amount, approveAmount) + // wait for the cctx to be mined cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.CctxTimeout) utils.RequireCCTXStatus(r, cctx, cctypes.CctxStatus_OutboundMined) diff --git a/e2e/runner/ton/deployer.go b/e2e/runner/ton/deployer.go index 4735d3be10..3f12f784da 100644 --- a/e2e/runner/ton/deployer.go +++ b/e2e/runner/ton/deployer.go @@ -57,10 +57,13 @@ func (d *Deployer) Seqno(ctx context.Context) (uint32, error) { return d.blockchain.GetSeqno(ctx, d.GetAddress()) } -// GetBalanceOf returns the balance of the given account. -func (d *Deployer) GetBalanceOf(ctx context.Context, id ton.AccountID) (math.Uint, error) { - if err := d.waitForAccountActivation(ctx, id); err != nil { - return math.Uint{}, errors.Wrap(err, "failed to wait for account activation") +// GetBalanceOf returns the balance of a given account. +// wait=true waits for account activation. +func (d *Deployer) GetBalanceOf(ctx context.Context, id ton.AccountID, wait bool) (math.Uint, error) { + if wait { + if err := d.waitForAccountActivation(ctx, id); err != nil { + return math.Uint{}, errors.Wrap(err, "failed to wait for account activation") + } } state, err := d.blockchain.GetAccountState(ctx, id) diff --git a/e2e/runner/zeta.go b/e2e/runner/zeta.go index 1df7e676e3..d59ef85645 100644 --- a/e2e/runner/zeta.go +++ b/e2e/runner/zeta.go @@ -99,11 +99,15 @@ func (r *E2ERunner) WaitForMinedCCTX(txHash ethcommon.Hash) { // WaitForMinedCCTXFromIndex waits for a cctx to be mined from its index func (r *E2ERunner) WaitForMinedCCTXFromIndex(index string) *types.CrossChainTx { + return r.waitForMinedCCTXFromIndex(index, types.CctxStatus_OutboundMined) +} + +func (r *E2ERunner) waitForMinedCCTXFromIndex(index string, status types.CctxStatus) *types.CrossChainTx { r.Lock() defer r.Unlock() cctx := utils.WaitCCTXMinedByIndex(r.Ctx, index, r.CctxClient, r.Logger, r.CctxTimeout) - utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined) + utils.RequireCCTXStatus(r, cctx, status) return cctx } @@ -111,6 +115,7 @@ func (r *E2ERunner) WaitForMinedCCTXFromIndex(index string) *types.CrossChainTx // WaitForSpecificCCTX scans for cctx by filters and ensures it's mined func (r *E2ERunner) WaitForSpecificCCTX( filter func(*types.CrossChainTx) bool, + status types.CctxStatus, timeout time.Duration, ) *types.CrossChainTx { var ( @@ -128,7 +133,7 @@ func (r *E2ERunner) WaitForSpecificCCTX( for i := range res.CrossChainTx { tx := res.CrossChainTx[i] if filter(tx) { - return r.WaitForMinedCCTXFromIndex(tx.Index) + return r.waitForMinedCCTXFromIndex(tx.Index, status) } } diff --git a/pkg/contracts/ton/gateway_msg.go b/pkg/contracts/ton/gateway_msg.go index 543f7d5a22..3a4716e0b4 100644 --- a/pkg/contracts/ton/gateway_msg.go +++ b/pkg/contracts/ton/gateway_msg.go @@ -24,6 +24,15 @@ const ( const OpWithdraw Op = 200 +// ExitCode represents an error code. Might be TVM or custom. +// TVM: https://docs.ton.org/v3/documentation/tvm/tvm-exit-codes +// Zeta: https://github.com/zeta-chain/protocol-contracts-ton/blob/main/contracts/common/errors.fc +type ExitCode uint32 + +const ( + ExitCodeInvalidSeqno ExitCode = 109 +) + // Donation represents a donation operation type Donation struct { Sender ton.AccountID diff --git a/zetaclient/chains/ton/observer/inbound.go b/zetaclient/chains/ton/observer/inbound.go index 881853241a..092bfa0e9d 100644 --- a/zetaclient/chains/ton/observer/inbound.go +++ b/zetaclient/chains/ton/observer/inbound.go @@ -19,32 +19,40 @@ import ( ) const ( - // MaxTransactionsPerTick is the maximum number of transactions to process on a ticker - MaxTransactionsPerTick = 100 + // maximum number of transactions to process on a ticker + // TODO: move to config + // https://github.com/zeta-chain/node/issues/3086 + maxTransactionsPerTick = 100 + // zero log sample rate for sampled logger (to avoid spamming logs) + logSampleRate = 10 ) // watchInbound watches for new txs to Gateway's account. func (ob *Observer) watchInbound(ctx context.Context) error { + return ob.inboundTicker(ctx, "WatchInbound", ob.observeGateway) +} + +func (ob *Observer) watchInboundTracker(ctx context.Context) error { + return ob.inboundTicker(ctx, "WatchInboundTracker", ob.processInboundTrackers) +} + +func (ob *Observer) inboundTicker(ctx context.Context, taskName string, taskFunc func(context.Context) error) error { app, err := zctx.FromContext(ctx) if err != nil { return err } - var ( - initialInterval = ticker.DurationFromUint64Seconds(ob.ChainParams().InboundTicker) - sampledLogger = ob.Logger().Inbound.Sample(&zerolog.BasicSampler{N: 10}) - ) - - ob.Logger().Inbound.Info().Msgf("WatchInbound started") + initialInterval := ticker.DurationFromUint64Seconds(ob.ChainParams().InboundTicker) + sampledLogger := ob.Logger().Inbound.Sample(&zerolog.BasicSampler{N: logSampleRate}) task := func(ctx context.Context, t *ticker.Ticker) error { if !app.IsInboundObservationEnabled() { - sampledLogger.Info().Msg("WatchInbound: inbound observation is disabled") + sampledLogger.Info().Msgf("%s: inbound observation is disabled", taskName) return nil } - if err := ob.observeGateway(ctx); err != nil { - ob.Logger().Inbound.Err(err).Msg("WatchInbound: observeGateway error") + if err := taskFunc(ctx); err != nil { + ob.Logger().Inbound.Err(err).Msgf("%s failed", taskName) } newInterval := ticker.DurationFromUint64Seconds(ob.ChainParams().InboundTicker) @@ -58,7 +66,7 @@ func (ob *Observer) watchInbound(ctx context.Context) error { initialInterval, task, ticker.WithStopChan(ob.StopChannel()), - ticker.WithLogger(ob.Logger().Inbound, "WatchInbound"), + ticker.WithLogger(ob.Logger().Inbound, taskName), ) } @@ -86,11 +94,11 @@ func (ob *Observer) observeGateway(ctx context.Context) error { case len(txs) == 0: // noop return nil - case len(txs) > MaxTransactionsPerTick: + case len(txs) > maxTransactionsPerTick: ob.Logger().Inbound.Info(). - Msgf("observeGateway: got %d transactions. Taking first %d", len(txs), MaxTransactionsPerTick) + Msgf("observeGateway: got %d transactions. Taking first %d", len(txs), maxTransactionsPerTick) - txs = txs[:MaxTransactionsPerTick] + txs = txs[:maxTransactionsPerTick] default: ob.Logger().Inbound.Info().Msgf("observeGateway: got %d transactions", len(txs)) } @@ -156,6 +164,63 @@ func (ob *Observer) observeGateway(ctx context.Context) error { return nil } +// processInboundTrackers handles adhoc trackers that were somehow missed by +func (ob *Observer) processInboundTrackers(ctx context.Context) error { + trackers, err := ob.ZetacoreClient().GetInboundTrackersForChain(ctx, ob.Chain().ChainId) + if err != nil { + return errors.Wrap(err, "unable to get inbound trackers") + } + + // noop + if len(trackers) == 0 { + return nil + } + + gatewayAccountID := ob.gateway.AccountID() + + // a single error should not block other trackers + for _, tracker := range trackers { + txHash := tracker.TxHash + + lt, hash, err := liteapi.TransactionHashFromString(txHash) + if err != nil { + ob.logSkippedTracker(txHash, "unable_to_parse_hash", err) + continue + } + + raw, err := ob.client.GetTransaction(ctx, gatewayAccountID, lt, hash) + if err != nil { + ob.logSkippedTracker(txHash, "unable_to_get_tx", err) + continue + } + + tx, err := ob.gateway.ParseTransaction(raw) + + switch { + case errors.Is(err, toncontracts.ErrParse) || errors.Is(err, toncontracts.ErrUnknownOp): + ob.logSkippedTracker(txHash, "unrelated_tx", err) + continue + case err != nil: + // should not happen + ob.logSkippedTracker(txHash, "unexpected_error", err) + continue + case tx.ExitCode != 0: + ob.logSkippedTracker(txHash, "failed_tx", nil) + continue + case tx.IsOutbound(): + ob.logSkippedTracker(txHash, "outbound_tx", nil) + continue + } + + if _, err := ob.voteInbound(ctx, tx); err != nil { + ob.logSkippedTracker(txHash, "vote_failed", err) + continue + } + } + + return nil +} + // Sends PostVoteInbound to zetacore func (ob *Observer) voteInbound(ctx context.Context, tx *toncontracts.Transaction) (string, error) { // noop @@ -283,6 +348,14 @@ func (ob *Observer) setLastScannedTX(tx *toncontracts.Transaction) { Msg("setLastScannedTX: WriteLastTxScannedToDB") } +func (ob *Observer) logSkippedTracker(hash string, reason string, err error) { + ob.Logger().Inbound.Warn(). + Str("transaction.hash", hash). + Str("skip_reason", reason). + Err(err). + Msg("Skipping tracker") +} + func txLogFields(tx *toncontracts.Transaction) map[string]any { return map[string]any{ "transaction.hash": liteapi.TransactionToHashString(tx.Transaction), diff --git a/zetaclient/chains/ton/observer/inbound_test.go b/zetaclient/chains/ton/observer/inbound_test.go index 953db3c744..e0b7478cfa 100644 --- a/zetaclient/chains/ton/observer/inbound_test.go +++ b/zetaclient/chains/ton/observer/inbound_test.go @@ -2,6 +2,7 @@ package observer import ( "encoding/hex" + "fmt" "testing" "github.com/pkg/errors" @@ -11,14 +12,11 @@ import ( "github.com/tonkeeper/tongo/ton" toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" "github.com/zeta-chain/node/testutil/sample" + cc "github.com/zeta-chain/node/x/crosschain/types" "github.com/zeta-chain/node/zetaclient/chains/ton/liteapi" ) func TestInbound(t *testing.T) { - gw := toncontracts.NewGateway( - ton.MustParseAccountID("0:997d889c815aeac21c47f86ae0e38383efc3c3463067582f6263ad48c5a1485b"), - ) - t.Run("No gateway provided", func(t *testing.T) { ts := newTestSuite(t) @@ -32,11 +30,11 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) // Given mocked lite client call - ts.OnGetFirstTransaction(gw.AccountID(), nil, 0, errors.New("oops")).Once() + ts.OnGetFirstTransaction(ts.gateway.AccountID(), nil, 0, errors.New("oops")).Once() // ACT // Observe inbounds once @@ -52,16 +50,16 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given mocked lite client calls - firstTX := sample.TONDonation(t, gw.AccountID(), toncontracts.Donation{ + firstTX := sample.TONDonation(t, ts.gateway.AccountID(), toncontracts.Donation{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "1"), }) - ts.OnGetFirstTransaction(gw.AccountID(), &firstTX, 0, nil).Once() - ts.OnGetTransactionsSince(gw.AccountID(), firstTX.Lt, txHash(firstTX), nil, nil).Once() + ts.OnGetFirstTransaction(ts.gateway.AccountID(), &firstTX, 0, nil).Once() + ts.OnGetTransactionsSince(ts.gateway.AccountID(), firstTX.Lt, txHash(firstTX), nil, nil).Once() // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) // ACT @@ -88,13 +86,13 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls - donation := sample.TONDonation(t, gw.AccountID(), toncontracts.Donation{ + donation := sample.TONDonation(t, ts.gateway.AccountID(), toncontracts.Donation{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "12"), }) @@ -102,7 +100,7 @@ func TestInbound(t *testing.T) { txs := []ton.Transaction{donation} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() // ACT @@ -124,10 +122,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls deposit := toncontracts.Deposit{ @@ -136,11 +134,11 @@ func TestInbound(t *testing.T) { Recipient: sample.EthAddress(), } - depositTX := sample.TONDeposit(t, gw.AccountID(), deposit) + depositTX := sample.TONDeposit(t, ts.gateway.AccountID(), deposit) txs := []ton.Transaction{depositTX} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() ts.MockGetBlockHeader(depositTX.BlockID) @@ -182,10 +180,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls const callData = "hey there" @@ -198,11 +196,11 @@ func TestInbound(t *testing.T) { CallData: []byte(callData), } - depositAndCallTX := sample.TONDepositAndCall(t, gw.AccountID(), depositAndCall) + depositAndCallTX := sample.TONDepositAndCall(t, ts.gateway.AccountID(), depositAndCall) txs := []ton.Transaction{depositAndCallTX} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() ts.MockGetBlockHeader(depositAndCallTX.BlockID) @@ -251,10 +249,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given mocked lite client calls withdrawal := toncontracts.Withdrawal{ @@ -269,11 +267,11 @@ func TestInbound(t *testing.T) { require.NoError(t, err) require.Equal(t, ob.TSS().EVMAddress().Hex(), withdrawalSigner.Hex()) - withdrawalTX := sample.TONWithdrawal(t, gw.AccountID(), withdrawal) + withdrawalTX := sample.TONWithdrawal(t, ts.gateway.AccountID(), withdrawal) txs := []ton.Transaction{withdrawalTX} ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() // ACT @@ -299,10 +297,10 @@ func TestInbound(t *testing.T) { ts := newTestSuite(t) // Given observer - ob, err := New(ts.baseObserver, ts.liteClient, gw) + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) require.NoError(t, err) - lastScanned := ts.SetupLastScannedTX(gw.AccountID()) + lastScanned := ts.SetupLastScannedTX(ts.gateway.AccountID()) // Given several transactions withdrawal := toncontracts.Withdrawal{ @@ -314,39 +312,39 @@ func TestInbound(t *testing.T) { txs := []ton.Transaction{ // should be skipped - sample.TONDonation(t, gw.AccountID(), toncontracts.Donation{ + sample.TONDonation(t, ts.gateway.AccountID(), toncontracts.Donation{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "1"), }), // should be voted - sample.TONDeposit(t, gw.AccountID(), toncontracts.Deposit{ + sample.TONDeposit(t, ts.gateway.AccountID(), toncontracts.Deposit{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "3"), Recipient: sample.EthAddress(), }), // should be skipped (invalid inbound message) sample.TONTransaction(t, sample.TONTransactionProps{ - Account: gw.AccountID(), + Account: ts.gateway.AccountID(), Input: &tlb.Message{}, }), // should be voted - sample.TONDeposit(t, gw.AccountID(), toncontracts.Deposit{ + sample.TONDeposit(t, ts.gateway.AccountID(), toncontracts.Deposit{ Sender: sample.GenerateTONAccountID(), Amount: tonCoins(t, "3"), Recipient: sample.EthAddress(), }), // a tracker should be added - sample.TONWithdrawal(t, gw.AccountID(), withdrawal), + sample.TONWithdrawal(t, ts.gateway.AccountID(), withdrawal), // should be skipped (invalid inbound/outbound messages) sample.TONTransaction(t, sample.TONTransactionProps{ - Account: gw.AccountID(), + Account: ts.gateway.AccountID(), Input: &tlb.Message{}, Output: &tlb.Message{}, }), } ts. - OnGetTransactionsSince(gw.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). + OnGetTransactionsSince(ts.gateway.AccountID(), lastScanned.Lt, txHash(lastScanned), txs, nil). Once() for _, tx := range txs { @@ -391,6 +389,58 @@ func TestInbound(t *testing.T) { }) } +func TestInboundTracker(t *testing.T) { + // ARRANGE + ts := newTestSuite(t) + + // Given observer + ob, err := New(ts.baseObserver, ts.liteClient, ts.gateway) + require.NoError(t, err) + + // Given TON gateway transactions + // should be voted + deposit := toncontracts.Deposit{ + Sender: sample.GenerateTONAccountID(), + Amount: toncontracts.Coins(5), + Recipient: sample.EthAddress(), + } + + txDeposit := sample.TONDeposit(t, ts.gateway.AccountID(), deposit) + ts.MockGetTransaction(ts.gateway.AccountID(), txDeposit) + ts.MockGetBlockHeader(txDeposit.BlockID) + + // Should be skipped (I doubt anyone would vote for this gov proposal, but let’s still put up rail guards) + txWithdrawal := sample.TONWithdrawal(t, ts.gateway.AccountID(), toncontracts.Withdrawal{ + Recipient: sample.GenerateTONAccountID(), + Amount: toncontracts.Coins(5), + Seqno: 1, + }) + ts.MockGetTransaction(ts.gateway.AccountID(), txWithdrawal) + ts.MockGetBlockHeader(txWithdrawal.BlockID) + + // Given inbound trackers from zetacore + trackers := []cc.InboundTracker{ + ts.TxToInboundTracker(txDeposit), + ts.TxToInboundTracker(txWithdrawal), + } + + ts.OnGetInboundTrackersForChain(trackers).Once() + + // ACT + err = ob.processInboundTrackers(ts.ctx) + + // ARRANGE + require.NoError(t, err) + require.Len(t, ts.votesBag, 1) + + vote := ts.votesBag[0] + assert.Equal(t, deposit.Amount, vote.Amount) + assert.Equal(t, deposit.Sender.ToRaw(), vote.Sender) + + // zevm recipient bytes == memo bytes + assert.Equal(t, fmt.Sprintf("%x", deposit.Recipient), vote.Message) +} + func txHash(tx ton.Transaction) ton.Bits256 { return ton.Bits256(tx.Hash()) } diff --git a/zetaclient/chains/ton/observer/observer.go b/zetaclient/chains/ton/observer/observer.go index 41523d5759..565a544073 100644 --- a/zetaclient/chains/ton/observer/observer.go +++ b/zetaclient/chains/ton/observer/observer.go @@ -86,17 +86,16 @@ func (ob *Observer) Start(ctx context.Context) { ob.Logger().Chain.Info().Msg("observer is starting") - start := func(job func(ctx context.Context) error, name string, log zerolog.Logger) { - bg.Work(ctx, job, bg.WithName(name), bg.WithLogger(log)) - } - - // TODO: watchInboundTracker - // https://github.com/zeta-chain/node/issues/2935 + start(ctx, ob.watchInbound, "WatchInbound", ob.Logger().Inbound) + start(ctx, ob.watchInboundTracker, "WatchInboundTracker", ob.Logger().Inbound) + start(ctx, ob.watchOutbound, "WatchOutbound", ob.Logger().Outbound) + start(ctx, ob.watchGasPrice, "WatchGasPrice", ob.Logger().GasPrice) + start(ctx, ob.watchRPCStatus, "WatchRPCStatus", ob.Logger().Chain) +} - start(ob.watchInbound, "WatchInbound", ob.Logger().Inbound) - start(ob.watchOutbound, "WatchOutbound", ob.Logger().Outbound) - start(ob.watchGasPrice, "WatchGasPrice", ob.Logger().GasPrice) - start(ob.watchRPCStatus, "WatchRPCStatus", ob.Logger().Chain) +// fire goroutine task +func start(ctx context.Context, task func(ctx context.Context) error, name string, log zerolog.Logger) { + bg.Work(ctx, task, bg.WithName(name), bg.WithLogger(log)) } // watchGasPrice observes TON gas price and votes it to Zetacore. diff --git a/zetaclient/chains/ton/observer/observer_test.go b/zetaclient/chains/ton/observer/observer_test.go index 502a269267..290e34081a 100644 --- a/zetaclient/chains/ton/observer/observer_test.go +++ b/zetaclient/chains/ton/observer/observer_test.go @@ -13,6 +13,7 @@ import ( "github.com/tonkeeper/tongo/tlb" "github.com/tonkeeper/tongo/ton" "github.com/zeta-chain/node/pkg/chains" + "github.com/zeta-chain/node/pkg/coin" toncontracts "github.com/zeta-chain/node/pkg/contracts/ton" "github.com/zeta-chain/node/testutil/sample" cc "github.com/zeta-chain/node/x/crosschain/types" @@ -31,6 +32,7 @@ type testSuite struct { chain chains.Chain chainParams *observertypes.ChainParams + gateway *toncontracts.Gateway liteClient *mocks.LiteClient zetacore *mocks.ZetacoreClient @@ -55,6 +57,10 @@ func newTestSuite(t *testing.T) *testSuite { chain = chains.TONTestnet chainParams = sample.ChainParams(chain.ChainId) + gateway = toncontracts.NewGateway(ton.MustParseAccountID( + "0:997d889c815aeac21c47f86ae0e38383efc3c3463067582f6263ad48c5a1485b", + )) + liteClient = mocks.NewLiteClient(t) tss = mocks.NewGeneratedTSS(t, chain) @@ -90,6 +96,7 @@ func newTestSuite(t *testing.T) *testSuite { chainParams: chainParams, liteClient: liteClient, + gateway: gateway, zetacore: zetacore, tss: tss, @@ -168,6 +175,20 @@ func (ts *testSuite) MockGetBlockHeader(id ton.BlockIDExt) *mock.Call { Return(blockInfo, nil) } +func (ts *testSuite) OnGetInboundTrackersForChain(trackers []cc.InboundTracker) *mock.Call { + return ts.zetacore. + On("GetInboundTrackersForChain", mock.Anything, ts.chain.ChainId). + Return(trackers, nil) +} + +func (ts *testSuite) TxToInboundTracker(tx ton.Transaction) cc.InboundTracker { + return cc.InboundTracker{ + ChainId: ts.chain.ChainId, + TxHash: liteapi.TransactionToHashString(tx), + CoinType: coin.CoinType_Gas, + } +} + type signable interface { Hash() ([32]byte, error) SetSignature([65]byte) diff --git a/zetaclient/chains/ton/signer/signer.go b/zetaclient/chains/ton/signer/signer.go index 7542fb37f6..bdd25c0c18 100644 --- a/zetaclient/chains/ton/signer/signer.go +++ b/zetaclient/chains/ton/signer/signer.go @@ -2,10 +2,14 @@ package signer import ( "context" + "regexp" + "strconv" + "strings" ethcommon "github.com/ethereum/go-ethereum/common" lru "github.com/hashicorp/golang-lru" "github.com/pkg/errors" + "github.com/tonkeeper/tongo/liteclient" "github.com/tonkeeper/tongo/tlb" "github.com/tonkeeper/tongo/ton" @@ -84,13 +88,20 @@ func (s *Signer) TryProcessOutbound( }() outcome, err := s.ProcessOutbound(ctx, cctx, zetacore, zetaBlockHeight) - if err != nil { - s.Logger().Std.Error(). - Err(err). - Str("outbound.id", outboundID). - Uint64("outbound.nonce", cctx.GetCurrentOutboundParam().TssNonce). - Str("outbound.outcome", string(outcome)). - Msg("Unable to ProcessOutbound") + + lf := map[string]any{ + "outbound.id": outboundID, + "outbound.nonce": cctx.GetCurrentOutboundParam().TssNonce, + "outbound.outcome": string(outcome), + } + + switch { + case err != nil: + s.Logger().Std.Error().Err(err).Fields(lf).Msg("Unable to ProcessOutbound") + case outcome != Success: + s.Logger().Std.Warn().Fields(lf).Msg("Unsuccessful outcome for ProcessOutbound") + default: + s.Logger().Std.Info().Fields(lf).Msg("Processed outbound") } } @@ -147,16 +158,8 @@ func (s *Signer) ProcessOutbound( // Example: If a cctx has amount of 5 TON, the recipient will receive 5 TON, // and gateway's balance will be decreased by 5 TON + txFees. exitCode, err := s.gateway.SendExternalMessage(ctx, s.client, withdrawal) - switch { - case err != nil: - return Fail, errors.Wrap(err, "unable to send external message") - case exitCode != 0: - // Might happen if msg's nonce is too high; retry later. - lf["outbound.exit_code"] = exitCode - lf["outbound.outcome"] = string(Invalid) - s.Logger().Std.Info().Fields(lf).Msg("Unable to send external message") - - return Invalid, nil + if err != nil || exitCode != 0 { + return s.handleSendError(exitCode, err, lf) } // it's okay to run this in the same goroutine @@ -211,6 +214,62 @@ func (s *Signer) setSignature(hash [32]byte, sig [65]byte) { s.signaturesCache.Add(hash, sig) } +// Sample (from local ton): +// error code: 0 message: cannot apply external message to current state: +// External message was not accepted Cannot run message on account: +// inbound external message rejected by transaction ...: exitcode=109, steps=108, gas_used=0\ +// VM Log (truncated): ... +var exitCodeErrorRegex = regexp.MustCompile(`exitcode=(\d+)`) + +// handleSendError tries to figure out the reason of the send error. +func (s *Signer) handleSendError(exitCode uint32, err error, logFields map[string]any) (Outcome, error) { + if err != nil { + // Might be possible if 2 concurrent zeta clients + // are trying to broadcast the same message. + if strings.Contains(err.Error(), "duplicate message") { + s.Logger().Std.Warn().Fields(logFields).Msg("Message already sent") + return Invalid, nil + } + + var errLiteClient liteclient.LiteServerErrorC + if errors.As(err, &errLiteClient) { + logFields["outbound.error.message"] = errLiteClient.Message + exitCode = errLiteClient.Code + } + + if code, ok := extractExitCode(err.Error()); ok { + exitCode = code + } + } + + switch { + case exitCode == uint32(toncontracts.ExitCodeInvalidSeqno): + // Might be possible if zeta clients send several seq. numbers concurrently. + // In the current implementation, Gateway supports only 1 nonce per block. + logFields["outbound.error.exit_code"] = exitCode + s.Logger().Std.Warn().Fields(logFields).Msg("Invalid nonce, retry later") + return Invalid, nil + case err != nil: + return Fail, errors.Wrap(err, "unable to send external message") + default: + return Fail, errors.Errorf("unable to send external message: exit code %d", exitCode) + } +} + +func extractExitCode(text string) (uint32, bool) { + match := exitCodeErrorRegex.FindStringSubmatch(text) + if len(match) < 2 { + return 0, false + } + + exitCode, err := strconv.ParseUint(match[1], 10, 32) + if err != nil { + return 0, false + } + + return uint32(exitCode), true +} + // GetGatewayAddress returns gateway address as raw TON address "0:ABC..." func (s *Signer) GetGatewayAddress() string { return s.gateway.AccountID().ToRaw() diff --git a/zetaclient/chains/ton/signer/signer_test.go b/zetaclient/chains/ton/signer/signer_test.go index 9ac85f281a..dcd78c9817 100644 --- a/zetaclient/chains/ton/signer/signer_test.go +++ b/zetaclient/chains/ton/signer/signer_test.go @@ -85,6 +85,31 @@ func TestSigner(t *testing.T) { require.Equal(t, liteapi.TransactionToHashString(withdrawalTX), tracker.hash) } +func TestExitCodeRegex(t *testing.T) { + for _, tt := range []string{ + `unable to send external message: error code: 0 message: + cannot apply external message to current state : + External message was not accepted\nCannot run message on account: inbound external message rejected by + transaction CC8803E21EDA7E6487D191380725A82CD75316E1C131496E1A5636751CE60347: + \nexitcode=109, steps=108, gas_used=0\nVM Log (truncated):\n...INT 0\nexecute THROWIFNOT + 105\nexecute MYADDR\nexecute XCHG s1,s4\nexecute SDEQ\nexecute THROWIF 112\nexecute OVER\nexecute + EQINT 0\nexecute THROWIF 106\nexecute GETGLOB + 3\nexecute NEQ\nexecute THROWIF 109\ndefault exception handler, terminating vm with exit code 109\n`, + + `unable to send external message: error code: 0 message: cannot apply external message to current state : + External message was not accepted\nCannot run message on account: + inbound external message rejected by transaction + 6CCBB83C7D9BFBFDB40541F35AD069714856F18B4850C1273A117DF6BFADE1C6:\nexitcode=109, steps=108, + gas_used=0\nVM Log (truncated):\n...INT 0....`, + } { + require.True(t, exitCodeErrorRegex.MatchString(tt)) + + exitCode, ok := extractExitCode(tt) + require.True(t, ok) + require.Equal(t, uint32(109), exitCode) + } +} + type testSuite struct { ctx context.Context t *testing.T diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index b632b9234a..fe1c26b76a 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -756,7 +756,12 @@ func (oc *Orchestrator) ScheduleCCTXTON( } // fire async task - taskLogger := oc.logger.Logger.With().Str("outbound.id", outboundID).Logger() + taskLogger := oc.logger.Logger.With(). + Int64(logs.FieldChain, chainID). + Str("outbound.id", outboundID). + Uint64("outbound.nonce", cctx.GetCurrentOutboundParam().TssNonce). + Logger() + bg.Work(ctx, task, bg.WithName("TryProcessOutbound"), bg.WithLogger(taskLogger)) } }