diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index d4676b5271..500ba8f443 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -24,7 +24,7 @@ jobs: - name: "INSTALL:NODEJS" uses: actions/setup-node@v4 with: - node-version: 16 + node-version: 20 - name: "START:LOCAL:NET:WITH:STATE" run: | @@ -102,7 +102,7 @@ jobs: artillery report report.json --output artillery_report.html - name: "UPLOAD:REPORT" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: artillery-report diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0056022622..d11e422b21 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -94,6 +94,7 @@ jobs: core.setOutput('SOLANA_TESTS', labels.includes('SOLANA_TESTS')); } else if (context.eventName === 'merge_group') { core.setOutput('DEFAULT_TESTS', true); + core.setOutput('UPGRADE_LIGHT_TESTS', true); } else if (context.eventName === 'push' && context.ref === 'refs/heads/develop') { core.setOutput('DEFAULT_TESTS', true); } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) { @@ -112,6 +113,7 @@ jobs: core.setOutput('ADMIN_TESTS', true); core.setOutput('PERFORMANCE_TESTS', true); core.setOutput('STATEFUL_DATA_TESTS', true); + core.setOutput('TSS_MIGRATION_TESTS', true); core.setOutput('SOLANA_TESTS', true); } else if (context.eventName === 'workflow_dispatch') { core.setOutput('DEFAULT_TESTS', context.payload.inputs['default-test']); @@ -197,6 +199,7 @@ jobs: const cleanName = job.name.split("/")[0]; return `${icon} ${cleanName}`; }); + e2eResults.sort(); const overallResultStr = '${{ needs.e2e.result }}'; const overallResultPassing = overallResultStr === 'success' || overallResultStr === 'skipped'; diff --git a/changelog.md b/changelog.md index 1cfa91d42d..f255139f81 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ * [2634](https://github.com/zeta-chain/node/pull/2634) - add support for EIP-1559 gas fees * [2597](https://github.com/zeta-chain/node/pull/2597) - Add generic rpc metrics to zetaclient * [2538](https://github.com/zeta-chain/node/pull/2538) - add background worker routines to shutdown zetaclientd when needed for tss migration +* [2644](https://github.com/zeta-chain/node/pull/2644) - add created_timestamp to cctx status ### Refactor @@ -133,6 +134,7 @@ * [2481](https://github.com/zeta-chain/node/pull/2481) - increase gas limit inbound and outbound vote message to 500k * [2545](https://github.com/zeta-chain/node/pull/2545) - check solana minimum rent exempt to avoid outbound failure * [2547](https://github.com/zeta-chain/node/pull/2547) - limit max txs in priority mempool +* [2628](https://github.com/zeta-chain/node/pull/2628) - avoid submitting invalid hashes to outbound tracker ### CI diff --git a/cmd/zetaclientd-supervisor/lib.go b/cmd/zetaclientd-supervisor/lib.go index e65782c6b1..e2e54ba7a6 100644 --- a/cmd/zetaclientd-supervisor/lib.go +++ b/cmd/zetaclientd-supervisor/lib.go @@ -67,7 +67,6 @@ type zetaclientdSupervisor struct { upgradesDir string upgradePlanName string enableAutoDownload bool - restartChan chan os.Signal } func newZetaclientdSupervisor( @@ -83,15 +82,12 @@ func newZetaclientdSupervisor( if err != nil { return nil, fmt.Errorf("grpc dial: %w", err) } - // these signals will result in the supervisor process only restarting zetaclientd - restartChan := make(chan os.Signal, 1) return &zetaclientdSupervisor{ zetacoredConn: conn, logger: logger, reloadSignals: make(chan bool, 1), upgradesDir: defaultUpgradesDir, enableAutoDownload: enableAutoDownload, - restartChan: restartChan, }, nil } diff --git a/cmd/zetaclientd-supervisor/main.go b/cmd/zetaclientd-supervisor/main.go index d7179d6948..4a219a544f 100644 --- a/cmd/zetaclientd-supervisor/main.go +++ b/cmd/zetaclientd-supervisor/main.go @@ -50,8 +50,6 @@ func main() { os.Exit(1) } supervisor.Start(ctx) - // listen for SIGHUP to trigger a restart of zetaclientd - signal.Notify(supervisor.restartChan, syscall.SIGHUP) shouldRestart := true for shouldRestart { @@ -81,6 +79,7 @@ func main() { }) eg.Go(func() error { supervisor.WaitForReloadSignal(ctx) + cancel() return nil }) eg.Go(func() error { @@ -88,8 +87,6 @@ func main() { select { case <-ctx.Done(): return nil - case sig := <-supervisor.restartChan: - logger.Info().Msgf("got signal %d, sending SIGINT to zetaclientd", sig) case sig := <-shutdownChan: logger.Info().Msgf("got signal %d, shutting down", sig) shouldRestart = false diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 65fafeb054..521c53569e 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -41,6 +41,7 @@ const ( flagTestTSSMigration = "test-tss-migration" flagSkipBitcoinSetup = "skip-bitcoin-setup" flagSkipHeaderProof = "skip-header-proof" + flagSkipTrackerCheck = "skip-tracker-check" ) var ( @@ -73,6 +74,7 @@ func NewLocalCmd() *cobra.Command { cmd.Flags().Bool(flagSkipBitcoinSetup, false, "set to true to skip bitcoin wallet setup") cmd.Flags().Bool(flagSkipHeaderProof, false, "set to true to skip header proof tests") cmd.Flags().Bool(flagTestTSSMigration, false, "set to true to include a migration test at the end") + cmd.Flags().Bool(flagSkipTrackerCheck, false, "set to true to skip tracker check at the end of the tests") return cmd } @@ -94,6 +96,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { skipSetup = must(cmd.Flags().GetBool(flagSkipSetup)) skipBitcoinSetup = must(cmd.Flags().GetBool(flagSkipBitcoinSetup)) skipHeaderProof = must(cmd.Flags().GetBool(flagSkipHeaderProof)) + skipTrackerCheck = must(cmd.Flags().GetBool(flagSkipTrackerCheck)) testTSSMigration = must(cmd.Flags().GetBool(flagTestTSSMigration)) ) @@ -355,7 +358,9 @@ func localE2ETest(cmd *cobra.Command, _ []string) { runTSSMigrationTest(deployerRunner, logger, verbose, conf) } // Verify that there are no trackers left over after tests complete - deployerRunner.EnsureNoTrackers() + if !skipTrackerCheck { + deployerRunner.EnsureNoTrackers() + } // print and validate report networkReport, err := deployerRunner.GenerateNetworkReport() if err != nil { diff --git a/contrib/localnet/orchestrator/start-zetae2e.sh b/contrib/localnet/orchestrator/start-zetae2e.sh index dda5b12cd7..5fa2f3e443 100644 --- a/contrib/localnet/orchestrator/start-zetae2e.sh +++ b/contrib/localnet/orchestrator/start-zetae2e.sh @@ -99,9 +99,11 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then # set upgrade height to 225 by default UPGRADE_HEIGHT=${UPGRADE_HEIGHT:=225} + OLD_VERSION=$(get_zetacored_version) + COMMON_ARGS="--skip-header-proof --skip-tracker-check" if [[ ! -f deployed.yml ]]; then - zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml --skip-header-proof + zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml ${COMMON_ARGS} if [ $? -ne 0 ]; then echo "e2e setup failed" exit 1 @@ -115,7 +117,7 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then echo "running E2E command to setup the networks and populate the state..." # Use light flag to ensure tests can complete before the upgrade height - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light --skip-header-proof + zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light ${COMMON_ARGS} if [ $? -ne 0 ]; then echo "first e2e failed" exit 1 @@ -123,9 +125,6 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then fi echo "Waiting for upgrade height..." - - OLD_VERSION=$(get_zetacored_version) - CURRENT_HEIGHT=0 WAIT_HEIGHT=$(( UPGRADE_HEIGHT - 1 )) # wait for upgrade height @@ -157,9 +156,9 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then # When the upgrade height is greater than 100 for upgrade test, the Bitcoin tests have been run once, therefore the Bitcoin wallet is already set up # Use light flag to skip advanced tests if [ "$UPGRADE_HEIGHT" -lt 100 ]; then - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light --skip-header-proof + zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light ${COMMON_ARGS} else - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-bitcoin-setup --light --skip-header-proof + zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-bitcoin-setup --light ${COMMON_ARGS} fi ZETAE2E_EXIT_CODE=$? diff --git a/docs/openapi/openapi.swagger.yaml b/docs/openapi/openapi.swagger.yaml index 863b8e971b..137c3d1205 100644 --- a/docs/openapi/openapi.swagger.yaml +++ b/docs/openapi/openapi.swagger.yaml @@ -58426,6 +58426,10 @@ definitions: format: int64 isAbortRefunded: type: boolean + created_timestamp: + type: string + format: int64 + description: when the CCTX was created. only populated on new transactions. zetacoreemissionsParams: type: object properties: diff --git a/e2e/utils/zetacore.go b/e2e/utils/zetacore.go index a10dc8d68b..9122860d75 100644 --- a/e2e/utils/zetacore.go +++ b/e2e/utils/zetacore.go @@ -20,7 +20,7 @@ const ( AdminPolicyName = "admin" OperationalPolicyName = "operational" - DefaultCctxTimeout = 4 * time.Minute + DefaultCctxTimeout = 6 * time.Minute ) // WaitCctxMinedByInboundHash waits until cctx is mined; returns the cctxIndex (the last one) diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index 886cb7ac45..57a922e5c0 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -1,6 +1,12 @@ package constant +import "time" + const ( + // ZetaBlockTime is the block time of the ZetaChain network + // It's a rough estimate that can be used in non-critical path to estimate the time of a block + ZetaBlockTime = 6000 * time.Millisecond + // DonationMessage is the message for donation transactions // Transaction sent to the TSS or ERC20 Custody address containing this message are considered as a donation DonationMessage = "I am rich!" diff --git a/proto/zetachain/zetacore/crosschain/cross_chain_tx.proto b/proto/zetachain/zetacore/crosschain/cross_chain_tx.proto index 88ca547482..c1df04e719 100644 --- a/proto/zetachain/zetacore/crosschain/cross_chain_tx.proto +++ b/proto/zetachain/zetacore/crosschain/cross_chain_tx.proto @@ -89,6 +89,8 @@ message Status { string status_message = 2; int64 lastUpdate_timestamp = 3; bool isAbortRefunded = 4; + // when the CCTX was created. only populated on new transactions. + int64 created_timestamp = 5; } message CrossChainTx { diff --git a/testutil/sample/crosschain.go b/testutil/sample/crosschain.go index 9a532c0fca..2894431724 100644 --- a/testutil/sample/crosschain.go +++ b/testutil/sample/crosschain.go @@ -209,10 +209,13 @@ func OutboundParamsValidChainID(r *rand.Rand) *types.OutboundParams { func Status(t *testing.T, index string) *types.Status { r := newRandFromStringSeed(t, index) + createdAt := r.Int63() + return &types.Status{ Status: types.CctxStatus(r.Intn(100)), StatusMessage: String(), - LastUpdateTimestamp: r.Int63(), + CreatedTimestamp: createdAt, + LastUpdateTimestamp: createdAt, } } diff --git a/typescript/zetachain/zetacore/crosschain/cross_chain_tx_pb.d.ts b/typescript/zetachain/zetacore/crosschain/cross_chain_tx_pb.d.ts index c4839a61f5..a6a8bd9057 100644 --- a/typescript/zetachain/zetacore/crosschain/cross_chain_tx_pb.d.ts +++ b/typescript/zetachain/zetacore/crosschain/cross_chain_tx_pb.d.ts @@ -315,6 +315,13 @@ export declare class Status extends Message { */ isAbortRefunded: boolean; + /** + * when the CCTX was created. only populated on new transactions. + * + * @generated from field: int64 created_timestamp = 5; + */ + createdTimestamp: bigint; + constructor(data?: PartialMessage); static readonly runtime: typeof proto3; diff --git a/x/crosschain/keeper/abci.go b/x/crosschain/keeper/abci.go index 494603f473..391b417fb0 100644 --- a/x/crosschain/keeper/abci.go +++ b/x/crosschain/keeper/abci.go @@ -171,7 +171,6 @@ func CheckAndUpdateCctxGasPrice( // set new gas price and last update timestamp cctx.GetCurrentOutboundParam().GasPrice = newGasPrice.String() cctx.GetCurrentOutboundParam().GasPriorityFee = newPriorityFee.String() - cctx.CctxStatus.LastUpdateTimestamp = ctx.BlockHeader().Time.Unix() k.SetCrossChainTx(ctx, cctx) return gasPriceIncrease, additionalFees, nil diff --git a/x/crosschain/keeper/abci_test.go b/x/crosschain/keeper/abci_test.go index 7ac38cd917..03da0573f6 100644 --- a/x/crosschain/keeper/abci_test.go +++ b/x/crosschain/keeper/abci_test.go @@ -128,6 +128,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "a1", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -152,6 +153,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "a2", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -180,6 +182,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "a3", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -208,6 +211,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "b0", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -235,6 +239,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "b1", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -257,6 +262,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "b2", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -279,6 +285,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "b3", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -301,6 +308,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "c1", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ @@ -322,6 +330,7 @@ func TestCheckAndUpdateCctxGasPrice(t *testing.T) { cctx: types.CrossChainTx{ Index: "c2", CctxStatus: &types.Status{ + CreatedTimestamp: sampleTimestamp.Unix(), LastUpdateTimestamp: sampleTimestamp.Unix(), }, OutboundParams: []*types.OutboundParams{ diff --git a/x/crosschain/keeper/cctx.go b/x/crosschain/keeper/cctx.go index b89b517653..306df6979b 100644 --- a/x/crosschain/keeper/cctx.go +++ b/x/crosschain/keeper/cctx.go @@ -56,6 +56,11 @@ func (k Keeper) SetCctxAndNonceToCctxAndInboundHashToCctx(ctx sdk.Context, cctx // SetCrossChainTx set a specific send in the store from its index func (k Keeper) SetCrossChainTx(ctx sdk.Context, cctx types.CrossChainTx) { + // only set the update timestamp if the block height is >0 to allow + // for a genesis import + if cctx.CctxStatus != nil && ctx.BlockHeight() > 0 { + cctx.CctxStatus.LastUpdateTimestamp = ctx.BlockHeader().Time.Unix() + } p := types.KeyPrefix(fmt.Sprintf("%s", types.CCTXKey)) store := prefix.NewStore(ctx.KVStore(k.storeKey), p) b := k.cdc.MustMarshal(&cctx) diff --git a/x/crosschain/keeper/grpc_query_cctx_test.go b/x/crosschain/keeper/grpc_query_cctx_test.go index 77e9bbc175..6112d5e58c 100644 --- a/x/crosschain/keeper/grpc_query_cctx_test.go +++ b/x/crosschain/keeper/grpc_query_cctx_test.go @@ -283,5 +283,8 @@ func TestKeeper_CctxByNonce(t *testing.T) { }) require.NoError(t, err) require.Equal(t, cctx, res.CrossChainTx) + + // ensure that LastUpdateTimestamp is set to current block time + require.Equal(t, res.CrossChainTx.CctxStatus.LastUpdateTimestamp, ctx.BlockTime().Unix()) }) } diff --git a/x/crosschain/keeper/msg_server_abort_stuck_cctx_test.go b/x/crosschain/keeper/msg_server_abort_stuck_cctx_test.go index cdc38b36f2..5e6df45d9b 100644 --- a/x/crosschain/keeper/msg_server_abort_stuck_cctx_test.go +++ b/x/crosschain/keeper/msg_server_abort_stuck_cctx_test.go @@ -74,6 +74,8 @@ func TestMsgServer_AbortStuckCCTX(t *testing.T) { require.True(t, found) require.Equal(t, crosschaintypes.CctxStatus_Aborted, cctxFound.CctxStatus.Status) require.Equal(t, crosschainkeeper.AbortMessage, cctxFound.CctxStatus.StatusMessage) + // ensure the last update timestamp is updated + require.Equal(t, cctxFound.CctxStatus.LastUpdateTimestamp, ctx.BlockTime().Unix()) }) t.Run("can abort a cctx in pending revert", func(t *testing.T) { diff --git a/x/crosschain/keeper/msg_server_refund_aborted_tx.go b/x/crosschain/keeper/msg_server_refund_aborted_tx.go index 201b507af6..fb37dd1c4c 100644 --- a/x/crosschain/keeper/msg_server_refund_aborted_tx.go +++ b/x/crosschain/keeper/msg_server_refund_aborted_tx.go @@ -72,7 +72,7 @@ func (k msgServer) RefundAbortedCCTX( commit() // set the cctx as refunded - cctx.CctxStatus.AbortRefunded(ctx.BlockTime().Unix()) + cctx.CctxStatus.AbortRefunded() k.SetCrossChainTx(ctx, cctx) diff --git a/x/crosschain/types/cctx.go b/x/crosschain/types/cctx.go index 6952a5092f..debd1656d6 100644 --- a/x/crosschain/types/cctx.go +++ b/x/crosschain/types/cctx.go @@ -122,7 +122,6 @@ func (m *CrossChainTx) AddOutbound( m.GetCurrentOutboundParam().EffectiveGasPrice = msg.ObservedOutboundEffectiveGasPrice m.GetCurrentOutboundParam().EffectiveGasLimit = msg.ObservedOutboundEffectiveGasLimit m.GetCurrentOutboundParam().ObservedExternalHeight = msg.ObservedOutboundBlockHeight - m.CctxStatus.LastUpdateTimestamp = ctx.BlockHeader().Time.Unix() return nil } @@ -211,6 +210,7 @@ func NewCCTX(ctx sdk.Context, msg MsgVoteInbound, tssPubkey string) (CrossChainT status := &Status{ Status: CctxStatus_PendingInbound, StatusMessage: "", + CreatedTimestamp: ctx.BlockHeader().Time.Unix(), LastUpdateTimestamp: ctx.BlockHeader().Time.Unix(), IsAbortRefunded: false, } diff --git a/x/crosschain/types/cctx_test.go b/x/crosschain/types/cctx_test.go index 57d09ed188..6a8a460335 100644 --- a/x/crosschain/types/cctx_test.go +++ b/x/crosschain/types/cctx_test.go @@ -199,7 +199,6 @@ func TestCrossChainTx_AddOutbound(t *testing.T) { require.Equal(t, cctx.GetCurrentOutboundParam().EffectiveGasPrice, sdkmath.NewInt(100)) require.Equal(t, cctx.GetCurrentOutboundParam().EffectiveGasLimit, uint64(20)) require.Equal(t, cctx.GetCurrentOutboundParam().ObservedExternalHeight, uint64(10)) - require.Equal(t, cctx.CctxStatus.LastUpdateTimestamp, ctx.BlockHeader().Time.Unix()) }) t.Run("successfully get outbound tx for failed ballot without amount check", func(t *testing.T) { @@ -220,7 +219,6 @@ func TestCrossChainTx_AddOutbound(t *testing.T) { require.Equal(t, cctx.GetCurrentOutboundParam().EffectiveGasPrice, sdkmath.NewInt(100)) require.Equal(t, cctx.GetCurrentOutboundParam().EffectiveGasLimit, uint64(20)) require.Equal(t, cctx.GetCurrentOutboundParam().ObservedExternalHeight, uint64(10)) - require.Equal(t, cctx.CctxStatus.LastUpdateTimestamp, ctx.BlockHeader().Time.Unix()) }) t.Run("failed to get outbound tx if amount does not match value received", func(t *testing.T) { diff --git a/x/crosschain/types/cross_chain_tx.pb.go b/x/crosschain/types/cross_chain_tx.pb.go index 5817c7b709..0f33b7e929 100644 --- a/x/crosschain/types/cross_chain_tx.pb.go +++ b/x/crosschain/types/cross_chain_tx.pb.go @@ -404,6 +404,8 @@ type Status struct { StatusMessage string `protobuf:"bytes,2,opt,name=status_message,json=statusMessage,proto3" json:"status_message,omitempty"` LastUpdateTimestamp int64 `protobuf:"varint,3,opt,name=lastUpdate_timestamp,json=lastUpdateTimestamp,proto3" json:"lastUpdate_timestamp,omitempty"` IsAbortRefunded bool `protobuf:"varint,4,opt,name=isAbortRefunded,proto3" json:"isAbortRefunded,omitempty"` + // when the CCTX was created. only populated on new transactions. + CreatedTimestamp int64 `protobuf:"varint,5,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` } func (m *Status) Reset() { *m = Status{} } @@ -467,6 +469,13 @@ func (m *Status) GetIsAbortRefunded() bool { return false } +func (m *Status) GetCreatedTimestamp() int64 { + if m != nil { + return m.CreatedTimestamp + } + return 0 +} + type CrossChainTx struct { Creator string `protobuf:"bytes,1,opt,name=creator,proto3" json:"creator,omitempty"` Index string `protobuf:"bytes,2,opt,name=index,proto3" json:"index,omitempty"` @@ -567,76 +576,77 @@ func init() { } var fileDescriptor_d4c1966807fb5cb2 = []byte{ - // 1100 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x72, 0x13, 0x47, - 0x10, 0xd6, 0x22, 0x21, 0x4b, 0xad, 0xbf, 0x65, 0x2c, 0x9c, 0x8d, 0x53, 0x08, 0x47, 0x29, 0x40, - 0x50, 0x41, 0x2a, 0xcc, 0x25, 0x95, 0x9b, 0xed, 0xc2, 0xe0, 0x10, 0xc0, 0xb5, 0x98, 0x1c, 0x38, - 0x64, 0x33, 0xda, 0x6d, 0xad, 0xa6, 0x2c, 0xed, 0x28, 0x3b, 0x23, 0xd7, 0x8a, 0xca, 0x43, 0x24, - 0xef, 0x90, 0x43, 0x8e, 0x79, 0x82, 0x9c, 0xb9, 0x85, 0x63, 0x2a, 0x07, 0x2a, 0x85, 0xdf, 0x20, - 0x4f, 0x90, 0x9a, 0x99, 0x5d, 0xc9, 0x72, 0xb9, 0x6c, 0x42, 0x72, 0x52, 0xf7, 0xd7, 0x33, 0x5f, - 0xb7, 0x7a, 0xbe, 0x9e, 0x59, 0xd8, 0x7c, 0x85, 0x92, 0xfa, 0x43, 0xca, 0xa2, 0x9e, 0xb6, 0x78, - 0x8c, 0x3d, 0x3f, 0xe6, 0x42, 0x18, 0x4c, 0x9b, 0x9e, 0xb6, 0x3d, 0x99, 0x74, 0x27, 0x31, 0x97, - 0x9c, 0x5c, 0x9b, 0xef, 0xe9, 0x66, 0x7b, 0xba, 0x8b, 0x3d, 0xeb, 0xcd, 0x90, 0x87, 0x5c, 0xaf, - 0xec, 0x29, 0xcb, 0x6c, 0x5a, 0xbf, 0x79, 0x46, 0xa2, 0xc9, 0x61, 0xd8, 0xf3, 0xb9, 0x4a, 0xc3, - 0x59, 0x64, 0xd6, 0xb5, 0x7f, 0x2d, 0x40, 0x6d, 0x2f, 0xea, 0xf3, 0x69, 0x14, 0xec, 0xd3, 0x98, - 0x8e, 0x05, 0x59, 0x83, 0xa2, 0xc0, 0x28, 0xc0, 0xd8, 0xb1, 0x36, 0xac, 0x4e, 0xd9, 0x4d, 0x3d, - 0x72, 0x13, 0x1a, 0xc6, 0x4a, 0xeb, 0x63, 0x81, 0x73, 0x69, 0xc3, 0xea, 0xe4, 0xdd, 0x9a, 0x81, - 0x77, 0x14, 0xba, 0x17, 0x90, 0x4f, 0xa0, 0x2c, 0x13, 0x8f, 0xc7, 0x2c, 0x64, 0x91, 0x93, 0xd7, - 0x14, 0x25, 0x99, 0x3c, 0xd3, 0x3e, 0xd9, 0x86, 0xb2, 0x4a, 0xee, 0xc9, 0xd9, 0x04, 0x9d, 0xc2, - 0x86, 0xd5, 0xa9, 0x6f, 0xde, 0xe8, 0x9e, 0xf1, 0xff, 0x26, 0x87, 0x61, 0x57, 0x57, 0xb9, 0xc3, - 0x59, 0x74, 0x30, 0x9b, 0xa0, 0x5b, 0xf2, 0x53, 0x8b, 0x34, 0xe1, 0x32, 0x15, 0x02, 0xa5, 0x73, - 0x59, 0x93, 0x1b, 0x87, 0x3c, 0x84, 0x22, 0x1d, 0xf3, 0x69, 0x24, 0x9d, 0xa2, 0x82, 0xb7, 0x7b, - 0xaf, 0xdf, 0x5e, 0xcf, 0xfd, 0xf9, 0xf6, 0xfa, 0xad, 0x90, 0xc9, 0xe1, 0xb4, 0xdf, 0xf5, 0xf9, - 0xb8, 0xe7, 0x73, 0x31, 0xe6, 0x22, 0xfd, 0xb9, 0x2b, 0x82, 0xc3, 0x9e, 0xaa, 0x43, 0x74, 0x5f, - 0xb0, 0x48, 0xba, 0xe9, 0x76, 0xf2, 0x19, 0xd4, 0x78, 0x5f, 0x60, 0x7c, 0x84, 0x81, 0x37, 0xa4, - 0x62, 0xe8, 0xac, 0xe8, 0x34, 0xd5, 0x0c, 0x7c, 0x44, 0xc5, 0x90, 0x7c, 0x01, 0xce, 0x7c, 0x11, - 0x26, 0x12, 0xe3, 0x88, 0x8e, 0xbc, 0x21, 0xb2, 0x70, 0x28, 0x9d, 0xd2, 0x86, 0xd5, 0x29, 0xb8, - 0x6b, 0x59, 0xfc, 0x41, 0x1a, 0x7e, 0xa4, 0xa3, 0xe4, 0x53, 0xa8, 0xf6, 0xe9, 0x68, 0xc4, 0xa5, - 0xc7, 0xa2, 0x00, 0x13, 0xa7, 0xac, 0xd9, 0x2b, 0x06, 0xdb, 0x53, 0x10, 0xd9, 0x84, 0xab, 0x03, - 0x16, 0xd1, 0x11, 0x7b, 0x85, 0x81, 0xa7, 0x5a, 0x92, 0x31, 0x83, 0x66, 0x5e, 0x9d, 0x07, 0x5f, - 0xa2, 0xa4, 0x29, 0x2d, 0x83, 0x35, 0x99, 0x78, 0x69, 0x84, 0x4a, 0xc6, 0x23, 0x4f, 0x48, 0x2a, - 0xa7, 0xc2, 0xa9, 0xe8, 0x2e, 0xdf, 0xef, 0x9e, 0xab, 0xa2, 0xee, 0x41, 0xb2, 0x7b, 0x62, 0xef, - 0x73, 0xbd, 0xd5, 0x6d, 0xca, 0x33, 0xd0, 0xf6, 0xf7, 0x50, 0x57, 0x89, 0xb7, 0x7c, 0x5f, 0xf5, - 0x8b, 0x45, 0x21, 0xf1, 0x60, 0x95, 0xf6, 0x79, 0x2c, 0xb3, 0x72, 0xd3, 0x83, 0xb0, 0x3e, 0xec, - 0x20, 0xae, 0xa4, 0x5c, 0x3a, 0x89, 0x66, 0x6a, 0xff, 0x54, 0x84, 0xfa, 0xb3, 0xa9, 0x3c, 0x29, - 0xd3, 0x75, 0x28, 0xc5, 0xe8, 0x23, 0x3b, 0x9a, 0x0b, 0x75, 0xee, 0x93, 0xdb, 0x60, 0x67, 0xb6, - 0x11, 0xeb, 0x5e, 0xa6, 0xd5, 0x46, 0x86, 0x67, 0x6a, 0x5d, 0x12, 0x64, 0xfe, 0xc3, 0x04, 0xb9, - 0x90, 0x5e, 0xe1, 0xbf, 0x49, 0x4f, 0x8d, 0x8e, 0x10, 0x5e, 0xc4, 0x23, 0x1f, 0xb5, 0xba, 0x0b, - 0x6e, 0x49, 0x0a, 0xf1, 0x54, 0xf9, 0x2a, 0x18, 0x52, 0xe1, 0x8d, 0xd8, 0x98, 0x19, 0x8d, 0x17, - 0xdc, 0x52, 0x48, 0xc5, 0xd7, 0xca, 0xcf, 0x82, 0x93, 0x98, 0xf9, 0x98, 0x0a, 0x56, 0x05, 0xf7, - 0x95, 0x4f, 0x3a, 0x60, 0xa7, 0x41, 0x1e, 0x33, 0x39, 0xf3, 0x06, 0x88, 0xce, 0x47, 0x7a, 0x4d, - 0xdd, 0xac, 0xd1, 0xf0, 0x2e, 0x22, 0x21, 0x50, 0xd0, 0x92, 0x2f, 0xe9, 0xa8, 0xb6, 0xdf, 0x47, - 0xb0, 0xe7, 0x4d, 0x03, 0x9c, 0x3b, 0x0d, 0x1f, 0x83, 0x2a, 0xd3, 0x9b, 0x0a, 0x0c, 0x9c, 0xa6, - 0x5e, 0xb9, 0x12, 0x52, 0xf1, 0x42, 0x60, 0x40, 0xbe, 0x85, 0x55, 0x1c, 0x0c, 0xd0, 0x97, 0xec, - 0x08, 0xbd, 0xc5, 0x9f, 0xbb, 0xaa, 0x5b, 0xdc, 0x4d, 0x5b, 0x7c, 0xf3, 0x3d, 0x5a, 0xbc, 0xa7, - 0x34, 0x35, 0xa7, 0x7a, 0x98, 0x75, 0xa5, 0x7b, 0x9a, 0xdf, 0x74, 0x76, 0x4d, 0x57, 0xb1, 0xb4, - 0xde, 0xb4, 0xf8, 0x1a, 0x80, 0x3a, 0x9c, 0xc9, 0xb4, 0x7f, 0x88, 0x33, 0x3d, 0x55, 0x65, 0x57, - 0x1d, 0xd7, 0xbe, 0x06, 0xce, 0x19, 0xc0, 0xea, 0xff, 0x3c, 0x80, 0x5f, 0x15, 0x4a, 0x35, 0xbb, - 0xd9, 0xfe, 0xdd, 0x82, 0xa2, 0x01, 0xc8, 0x16, 0x14, 0xd3, 0x5c, 0x96, 0xce, 0x75, 0xfb, 0x82, - 0x5c, 0x3b, 0xbe, 0x4c, 0xd2, 0x0c, 0xe9, 0x46, 0x72, 0x03, 0xea, 0xc6, 0xf2, 0xc6, 0x28, 0x04, - 0x0d, 0x51, 0x0f, 0x4c, 0xd9, 0xad, 0x19, 0xf4, 0x89, 0x01, 0xc9, 0x3d, 0x68, 0x8e, 0xa8, 0x90, - 0x2f, 0x26, 0x01, 0x95, 0xe8, 0x49, 0x36, 0x46, 0x21, 0xe9, 0x78, 0xa2, 0x27, 0x27, 0xef, 0xae, - 0x2e, 0x62, 0x07, 0x59, 0x88, 0x74, 0xa0, 0xc1, 0xc4, 0x96, 0x1a, 0x69, 0x17, 0x07, 0xd3, 0x28, - 0xc0, 0x40, 0x8f, 0x49, 0xc9, 0x3d, 0x0d, 0xb7, 0x7f, 0xcb, 0x43, 0x75, 0x47, 0x55, 0xa9, 0x87, - 0xf3, 0x20, 0x21, 0x0e, 0xac, 0xf8, 0x31, 0x52, 0xc9, 0xb3, 0x11, 0xcf, 0x5c, 0xf5, 0x06, 0x18, - 0x35, 0x9a, 0x2a, 0x8d, 0x43, 0xbe, 0x83, 0xb2, 0xbe, 0x7f, 0x06, 0x88, 0xc2, 0xbc, 0x0e, 0xdb, - 0x3b, 0xff, 0x72, 0x16, 0xff, 0x7e, 0x7b, 0xdd, 0x9e, 0xd1, 0xf1, 0xe8, 0xcb, 0xf6, 0x9c, 0xa9, - 0xed, 0x96, 0x94, 0xbd, 0x8b, 0x28, 0xc8, 0x2d, 0x68, 0xc4, 0x38, 0xa2, 0x33, 0x0c, 0xe6, 0x7d, - 0x2a, 0x9a, 0x49, 0x4a, 0xe1, 0xac, 0x51, 0xbb, 0x50, 0xf1, 0x7d, 0x99, 0x64, 0x1a, 0x50, 0x03, - 0x55, 0x39, 0xfb, 0x66, 0x39, 0x71, 0x2e, 0xe9, 0x99, 0x80, 0x3f, 0x3f, 0x1f, 0xf2, 0x1c, 0xea, - 0xcc, 0x3c, 0xcf, 0xde, 0x44, 0x5f, 0x7c, 0x7a, 0xfe, 0x2a, 0x9b, 0x9f, 0x5f, 0x40, 0xb5, 0xf4, - 0xa6, 0xbb, 0x35, 0xb6, 0xf4, 0xc4, 0x7f, 0x03, 0x0d, 0x9e, 0xde, 0xa6, 0x19, 0x2b, 0x6c, 0xe4, - 0x3b, 0x95, 0xcd, 0xbb, 0x17, 0xb0, 0x2e, 0xdf, 0xc1, 0x6e, 0x9d, 0x2f, 0xf9, 0x77, 0x7e, 0x00, - 0x58, 0x48, 0x8b, 0x10, 0xa8, 0xef, 0x63, 0x14, 0xb0, 0x28, 0x4c, 0x8b, 0xb1, 0x73, 0x64, 0x15, - 0x1a, 0x29, 0x96, 0x51, 0xd9, 0x16, 0xb9, 0x02, 0xb5, 0xcc, 0x7b, 0xc2, 0x22, 0x0c, 0xec, 0xbc, - 0x82, 0xd2, 0x75, 0x2e, 0x1e, 0x61, 0x2c, 0xed, 0x02, 0xa9, 0x42, 0xc9, 0xd8, 0x18, 0xd8, 0x97, - 0x49, 0x05, 0x56, 0xb6, 0xcc, 0x33, 0x61, 0x17, 0xd7, 0x0b, 0xbf, 0xfc, 0xdc, 0xb2, 0xee, 0x3c, - 0x86, 0xe6, 0x59, 0x43, 0x44, 0x6c, 0xa8, 0x3e, 0xe5, 0x72, 0x37, 0x7b, 0x34, 0xed, 0x1c, 0xa9, - 0x41, 0x79, 0xe1, 0x5a, 0x8a, 0xf9, 0x41, 0x82, 0xfe, 0x54, 0x91, 0x5d, 0x32, 0x64, 0xdb, 0x8f, - 0x5f, 0xbf, 0x6b, 0x59, 0x6f, 0xde, 0xb5, 0xac, 0xbf, 0xde, 0xb5, 0xac, 0x1f, 0x8f, 0x5b, 0xb9, - 0x37, 0xc7, 0xad, 0xdc, 0x1f, 0xc7, 0xad, 0xdc, 0xcb, 0x7b, 0x27, 0x94, 0xa4, 0x7a, 0x74, 0xf7, - 0xd4, 0x57, 0x56, 0x72, 0xf2, 0x83, 0x4e, 0x0b, 0xab, 0x5f, 0xd4, 0xdf, 0x5a, 0xf7, 0xff, 0x09, - 0x00, 0x00, 0xff, 0xff, 0xea, 0x6a, 0x48, 0x57, 0xfe, 0x09, 0x00, 0x00, + // 1119 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x72, 0x13, 0xc7, + 0x13, 0xd6, 0x22, 0x59, 0x96, 0x5a, 0xff, 0x96, 0xb1, 0xf0, 0x6f, 0x7f, 0x4e, 0x21, 0x1c, 0xa5, + 0x00, 0x41, 0x82, 0x54, 0x98, 0x4b, 0x2a, 0x37, 0xdb, 0x85, 0xc1, 0x21, 0x80, 0x6b, 0x31, 0x39, + 0x70, 0xc8, 0x66, 0xb4, 0xdb, 0x5a, 0x4d, 0x59, 0xda, 0x51, 0x76, 0x46, 0xae, 0x15, 0x95, 0x87, + 0x48, 0xde, 0x21, 0x87, 0x1c, 0xf3, 0x04, 0x39, 0x73, 0xe4, 0x98, 0xca, 0x81, 0x4a, 0xc1, 0x1b, + 0xe4, 0x9c, 0x43, 0x6a, 0x66, 0x76, 0x25, 0xcb, 0xe5, 0xb2, 0x09, 0xc9, 0x69, 0xbb, 0xbf, 0x9e, + 0xf9, 0x7a, 0xb6, 0xe7, 0xeb, 0x99, 0x81, 0xad, 0x97, 0x28, 0xa9, 0x3f, 0xa4, 0x2c, 0xea, 0x69, + 0x8b, 0xc7, 0xd8, 0xf3, 0x63, 0x2e, 0x84, 0xc1, 0xb4, 0xe9, 0x69, 0xdb, 0x93, 0x49, 0x77, 0x12, + 0x73, 0xc9, 0xc9, 0xd5, 0xf9, 0x9c, 0x6e, 0x36, 0xa7, 0xbb, 0x98, 0xb3, 0xd1, 0x0c, 0x79, 0xc8, + 0xf5, 0xc8, 0x9e, 0xb2, 0xcc, 0xa4, 0x8d, 0x1b, 0x67, 0x24, 0x9a, 0x1c, 0x85, 0x3d, 0x9f, 0xab, + 0x34, 0x9c, 0x45, 0x66, 0x5c, 0xfb, 0x97, 0x02, 0xd4, 0xf6, 0xa3, 0x3e, 0x9f, 0x46, 0xc1, 0x01, + 0x8d, 0xe9, 0x58, 0x90, 0x75, 0x28, 0x0a, 0x8c, 0x02, 0x8c, 0x1d, 0x6b, 0xd3, 0xea, 0x94, 0xdd, + 0xd4, 0x23, 0x37, 0xa0, 0x61, 0xac, 0x74, 0x7d, 0x2c, 0x70, 0x2e, 0x6d, 0x5a, 0x9d, 0xbc, 0x5b, + 0x33, 0xf0, 0xae, 0x42, 0xf7, 0x03, 0xf2, 0x11, 0x94, 0x65, 0xe2, 0xf1, 0x98, 0x85, 0x2c, 0x72, + 0xf2, 0x9a, 0xa2, 0x24, 0x93, 0xa7, 0xda, 0x27, 0x3b, 0x50, 0x56, 0xc9, 0x3d, 0x39, 0x9b, 0xa0, + 0x53, 0xd8, 0xb4, 0x3a, 0xf5, 0xad, 0xeb, 0xdd, 0x33, 0xfe, 0x6f, 0x72, 0x14, 0x76, 0xf5, 0x2a, + 0x77, 0x39, 0x8b, 0x0e, 0x67, 0x13, 0x74, 0x4b, 0x7e, 0x6a, 0x91, 0x26, 0xac, 0x50, 0x21, 0x50, + 0x3a, 0x2b, 0x9a, 0xdc, 0x38, 0xe4, 0x01, 0x14, 0xe9, 0x98, 0x4f, 0x23, 0xe9, 0x14, 0x15, 0xbc, + 0xd3, 0x7b, 0xf5, 0xe6, 0x5a, 0xee, 0xf7, 0x37, 0xd7, 0x6e, 0x86, 0x4c, 0x0e, 0xa7, 0xfd, 0xae, + 0xcf, 0xc7, 0x3d, 0x9f, 0x8b, 0x31, 0x17, 0xe9, 0xe7, 0x8e, 0x08, 0x8e, 0x7a, 0x6a, 0x1d, 0xa2, + 0xfb, 0x9c, 0x45, 0xd2, 0x4d, 0xa7, 0x93, 0x4f, 0xa0, 0xc6, 0xfb, 0x02, 0xe3, 0x63, 0x0c, 0xbc, + 0x21, 0x15, 0x43, 0x67, 0x55, 0xa7, 0xa9, 0x66, 0xe0, 0x43, 0x2a, 0x86, 0xe4, 0x73, 0x70, 0xe6, + 0x83, 0x30, 0x91, 0x18, 0x47, 0x74, 0xe4, 0x0d, 0x91, 0x85, 0x43, 0xe9, 0x94, 0x36, 0xad, 0x4e, + 0xc1, 0x5d, 0xcf, 0xe2, 0xf7, 0xd3, 0xf0, 0x43, 0x1d, 0x25, 0x1f, 0x43, 0xb5, 0x4f, 0x47, 0x23, + 0x2e, 0x3d, 0x16, 0x05, 0x98, 0x38, 0x65, 0xcd, 0x5e, 0x31, 0xd8, 0xbe, 0x82, 0xc8, 0x16, 0x5c, + 0x19, 0xb0, 0x88, 0x8e, 0xd8, 0x4b, 0x0c, 0x3c, 0x55, 0x92, 0x8c, 0x19, 0x34, 0xf3, 0xda, 0x3c, + 0xf8, 0x02, 0x25, 0x4d, 0x69, 0x19, 0xac, 0xcb, 0xc4, 0x4b, 0x23, 0x54, 0x32, 0x1e, 0x79, 0x42, + 0x52, 0x39, 0x15, 0x4e, 0x45, 0x57, 0xf9, 0x5e, 0xf7, 0x5c, 0x15, 0x75, 0x0f, 0x93, 0xbd, 0x13, + 0x73, 0x9f, 0xe9, 0xa9, 0x6e, 0x53, 0x9e, 0x81, 0xb6, 0xbf, 0x83, 0xba, 0x4a, 0xbc, 0xed, 0xfb, + 0xaa, 0x5e, 0x2c, 0x0a, 0x89, 0x07, 0x6b, 0xb4, 0xcf, 0x63, 0x99, 0x2d, 0x37, 0xdd, 0x08, 0xeb, + 0xc3, 0x36, 0xe2, 0x72, 0xca, 0xa5, 0x93, 0x68, 0xa6, 0xf6, 0x8f, 0x45, 0xa8, 0x3f, 0x9d, 0xca, + 0x93, 0x32, 0xdd, 0x80, 0x52, 0x8c, 0x3e, 0xb2, 0xe3, 0xb9, 0x50, 0xe7, 0x3e, 0xb9, 0x05, 0x76, + 0x66, 0x1b, 0xb1, 0xee, 0x67, 0x5a, 0x6d, 0x64, 0x78, 0xa6, 0xd6, 0x25, 0x41, 0xe6, 0x3f, 0x4c, + 0x90, 0x0b, 0xe9, 0x15, 0xfe, 0x9d, 0xf4, 0x54, 0xeb, 0x08, 0xe1, 0x45, 0x3c, 0xf2, 0x51, 0xab, + 0xbb, 0xe0, 0x96, 0xa4, 0x10, 0x4f, 0x94, 0xaf, 0x82, 0x21, 0x15, 0xde, 0x88, 0x8d, 0x99, 0xd1, + 0x78, 0xc1, 0x2d, 0x85, 0x54, 0x7c, 0xa5, 0xfc, 0x2c, 0x38, 0x89, 0x99, 0x8f, 0xa9, 0x60, 0x55, + 0xf0, 0x40, 0xf9, 0xa4, 0x03, 0x76, 0x1a, 0xe4, 0x31, 0x93, 0x33, 0x6f, 0x80, 0xe8, 0xfc, 0x4f, + 0x8f, 0xa9, 0x9b, 0x31, 0x1a, 0xde, 0x43, 0x24, 0x04, 0x0a, 0x5a, 0xf2, 0x25, 0x1d, 0xd5, 0xf6, + 0xfb, 0x08, 0xf6, 0xbc, 0x6e, 0x80, 0x73, 0xbb, 0xe1, 0xff, 0xa0, 0x96, 0xe9, 0x4d, 0x05, 0x06, + 0x4e, 0x53, 0x8f, 0x5c, 0x0d, 0xa9, 0x78, 0x2e, 0x30, 0x20, 0xdf, 0xc0, 0x1a, 0x0e, 0x06, 0xe8, + 0x4b, 0x76, 0x8c, 0xde, 0xe2, 0xe7, 0xae, 0xe8, 0x12, 0x77, 0xd3, 0x12, 0xdf, 0x78, 0x8f, 0x12, + 0xef, 0x2b, 0x4d, 0xcd, 0xa9, 0x1e, 0x64, 0x55, 0xe9, 0x9e, 0xe6, 0x37, 0x95, 0x5d, 0xd7, 0xab, + 0x58, 0x1a, 0x6f, 0x4a, 0x7c, 0x15, 0x40, 0x6d, 0xce, 0x64, 0xda, 0x3f, 0xc2, 0x99, 0xee, 0xaa, + 0xb2, 0xab, 0xb6, 0xeb, 0x40, 0x03, 0xe7, 0x34, 0x60, 0xf5, 0x3f, 0x6e, 0xc0, 0x2f, 0x0b, 0xa5, + 0x9a, 0xdd, 0x6c, 0xff, 0x65, 0x41, 0xd1, 0x00, 0x64, 0x1b, 0x8a, 0x69, 0x2e, 0x4b, 0xe7, 0xba, + 0x75, 0x41, 0xae, 0x5d, 0x5f, 0x26, 0x69, 0x86, 0x74, 0x22, 0xb9, 0x0e, 0x75, 0x63, 0x79, 0x63, + 0x14, 0x82, 0x86, 0xa8, 0x1b, 0xa6, 0xec, 0xd6, 0x0c, 0xfa, 0xd8, 0x80, 0xe4, 0x2e, 0x34, 0x47, + 0x54, 0xc8, 0xe7, 0x93, 0x80, 0x4a, 0xf4, 0x24, 0x1b, 0xa3, 0x90, 0x74, 0x3c, 0xd1, 0x9d, 0x93, + 0x77, 0xd7, 0x16, 0xb1, 0xc3, 0x2c, 0x44, 0x3a, 0xd0, 0x60, 0x62, 0x5b, 0xb5, 0xb4, 0x8b, 0x83, + 0x69, 0x14, 0x60, 0xa0, 0xdb, 0xa4, 0xe4, 0x9e, 0x86, 0xc9, 0xa7, 0x70, 0xd9, 0x8f, 0x91, 0xaa, + 0x63, 0x64, 0xc1, 0xbc, 0xa2, 0x99, 0xed, 0x34, 0x30, 0xa7, 0x6d, 0xff, 0x9a, 0x87, 0xea, 0xae, + 0xfa, 0x25, 0xdd, 0xc9, 0x87, 0x09, 0x71, 0x60, 0x55, 0x0f, 0xe2, 0xd9, 0x79, 0x90, 0xb9, 0xea, + 0xc2, 0x30, 0xd2, 0x35, 0xbf, 0x64, 0x1c, 0xf2, 0x2d, 0x94, 0xf5, 0x61, 0x35, 0x40, 0x14, 0xe6, + 0x2a, 0xd9, 0xd9, 0xfd, 0x87, 0x8d, 0xfb, 0xe7, 0x9b, 0x6b, 0xf6, 0x8c, 0x8e, 0x47, 0x5f, 0xb4, + 0xe7, 0x4c, 0x6d, 0xb7, 0xa4, 0xec, 0x3d, 0x44, 0x41, 0x6e, 0x42, 0x23, 0xc6, 0x11, 0x9d, 0x61, + 0x30, 0x2f, 0x6a, 0xd1, 0xb4, 0x5d, 0x0a, 0x67, 0x55, 0xdd, 0x83, 0x8a, 0xef, 0xcb, 0x24, 0x13, + 0x8c, 0xea, 0xbe, 0xca, 0xd9, 0xc7, 0xd0, 0x89, 0x4d, 0x4c, 0x37, 0x10, 0xfc, 0xf9, 0x66, 0x92, + 0x67, 0x50, 0x67, 0xe6, 0x2e, 0xf7, 0x26, 0xfa, 0x94, 0xd4, 0xcd, 0x5a, 0xd9, 0xfa, 0xec, 0x02, + 0xaa, 0xa5, 0x07, 0x80, 0x5b, 0x63, 0x4b, 0xef, 0x81, 0xaf, 0xa1, 0xc1, 0xd3, 0xa3, 0x37, 0x63, + 0x85, 0xcd, 0x7c, 0xa7, 0xb2, 0x75, 0xe7, 0x02, 0xd6, 0xe5, 0x03, 0xdb, 0xad, 0xf3, 0x25, 0xff, + 0xf6, 0xf7, 0x00, 0x0b, 0x1d, 0x12, 0x02, 0xf5, 0x03, 0x8c, 0x02, 0x16, 0x85, 0xe9, 0x62, 0xec, + 0x1c, 0x59, 0x83, 0x46, 0x8a, 0x65, 0x54, 0xb6, 0x45, 0x2e, 0x43, 0x2d, 0xf3, 0x1e, 0xb3, 0x08, + 0x03, 0x3b, 0xaf, 0xa0, 0x74, 0x9c, 0x8b, 0xc7, 0x18, 0x4b, 0xbb, 0x40, 0xaa, 0x50, 0x32, 0x36, + 0x06, 0xf6, 0x0a, 0xa9, 0xc0, 0xea, 0xb6, 0xb9, 0x53, 0xec, 0xe2, 0x46, 0xe1, 0xe7, 0x9f, 0x5a, + 0xd6, 0xed, 0x47, 0xd0, 0x3c, 0xab, 0xe3, 0x88, 0x0d, 0xd5, 0x27, 0x5c, 0xee, 0x65, 0x37, 0xac, + 0x9d, 0x23, 0x35, 0x28, 0x2f, 0x5c, 0x4b, 0x31, 0xdf, 0x4f, 0xd0, 0x9f, 0x2a, 0xb2, 0x4b, 0x86, + 0x6c, 0xe7, 0xd1, 0xab, 0xb7, 0x2d, 0xeb, 0xf5, 0xdb, 0x96, 0xf5, 0xc7, 0xdb, 0x96, 0xf5, 0xc3, + 0xbb, 0x56, 0xee, 0xf5, 0xbb, 0x56, 0xee, 0xb7, 0x77, 0xad, 0xdc, 0x8b, 0xbb, 0x27, 0x94, 0xa4, + 0x6a, 0x74, 0xe7, 0xd4, 0x93, 0x2c, 0x39, 0xf9, 0xfa, 0xd3, 0xc2, 0xea, 0x17, 0xf5, 0xc3, 0xec, + 0xde, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x84, 0xca, 0xb3, 0xff, 0x2b, 0x0a, 0x00, 0x00, } func (m *InboundParams) Marshal() (dAtA []byte, err error) { @@ -918,6 +928,11 @@ func (m *Status) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.CreatedTimestamp != 0 { + i = encodeVarintCrossChainTx(dAtA, i, uint64(m.CreatedTimestamp)) + i-- + dAtA[i] = 0x28 + } if m.IsAbortRefunded { i-- if m.IsAbortRefunded { @@ -1188,6 +1203,9 @@ func (m *Status) Size() (n int) { if m.IsAbortRefunded { n += 2 } + if m.CreatedTimestamp != 0 { + n += 1 + sovCrossChainTx(uint64(m.CreatedTimestamp)) + } return n } @@ -2238,6 +2256,25 @@ func (m *Status) Unmarshal(dAtA []byte) error { } } m.IsAbortRefunded = bool(v != 0) + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CreatedTimestamp", wireType) + } + m.CreatedTimestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCrossChainTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CreatedTimestamp |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipCrossChainTx(dAtA[iNdEx:]) diff --git a/x/crosschain/types/status.go b/x/crosschain/types/status.go index 320209ae78..cdaf5b0aae 100644 --- a/x/crosschain/types/status.go +++ b/x/crosschain/types/status.go @@ -4,10 +4,9 @@ import ( "fmt" ) -func (m *Status) AbortRefunded(timeStamp int64) { +func (m *Status) AbortRefunded() { m.IsAbortRefunded = true m.StatusMessage = "CCTX aborted and Refunded" - m.LastUpdateTimestamp = timeStamp } // ChangeStatus changes the status of the cross chain transaction diff --git a/x/crosschain/types/status_test.go b/x/crosschain/types/status_test.go index 247cc19f50..49ed1fc74f 100644 --- a/x/crosschain/types/status_test.go +++ b/x/crosschain/types/status_test.go @@ -3,7 +3,6 @@ package types_test import ( "fmt" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -19,11 +18,9 @@ func TestStatus_AbortRefunded(t *testing.T) { LastUpdateTimestamp: 0, IsAbortRefunded: false, } - timestamp := time.Now().Unix() - status.AbortRefunded(timestamp) + status.AbortRefunded() require.Equal(t, status.IsAbortRefunded, true) require.Equal(t, status.StatusMessage, "CCTX aborted and Refunded") - require.Equal(t, status.LastUpdateTimestamp, timestamp) }) } diff --git a/zetaclient/chains/base/observer.go b/zetaclient/chains/base/observer.go index 6021b9219e..cf5491366b 100644 --- a/zetaclient/chains/base/observer.go +++ b/zetaclient/chains/base/observer.go @@ -17,6 +17,7 @@ import ( observertypes "github.com/zeta-chain/zetacore/x/observer/types" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" "github.com/zeta-chain/zetacore/zetaclient/db" + "github.com/zeta-chain/zetacore/zetaclient/logs" "github.com/zeta-chain/zetacore/zetaclient/metrics" clienttypes "github.com/zeta-chain/zetacore/zetaclient/types" "github.com/zeta-chain/zetacore/zetaclient/zetacore" @@ -301,13 +302,13 @@ func (ob *Observer) Logger() *ObserverLogger { // WithLogger attaches a new logger to the observer. func (ob *Observer) WithLogger(logger Logger) *Observer { - chainLogger := logger.Std.With().Int64("chain", ob.chain.ChainId).Logger() + chainLogger := logger.Std.With().Int64(logs.FieldChain, ob.chain.ChainId).Logger() ob.logger = ObserverLogger{ Chain: chainLogger, - Inbound: chainLogger.With().Str("module", "inbound").Logger(), - Outbound: chainLogger.With().Str("module", "outbound").Logger(), - GasPrice: chainLogger.With().Str("module", "gasprice").Logger(), - Headers: chainLogger.With().Str("module", "headers").Logger(), + Inbound: chainLogger.With().Str(logs.FieldModule, logs.ModNameInbound).Logger(), + Outbound: chainLogger.With().Str(logs.FieldModule, logs.ModNameOutbound).Logger(), + GasPrice: chainLogger.With().Str(logs.FieldModule, logs.ModNameGasPrice).Logger(), + Headers: chainLogger.With().Str(logs.FieldModule, logs.ModNameHeaders).Logger(), Compliance: logger.Compliance, } return ob diff --git a/zetaclient/chains/base/signer.go b/zetaclient/chains/base/signer.go index 6618c338de..781288b513 100644 --- a/zetaclient/chains/base/signer.go +++ b/zetaclient/chains/base/signer.go @@ -5,6 +5,7 @@ import ( "github.com/zeta-chain/zetacore/pkg/chains" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" + "github.com/zeta-chain/zetacore/zetaclient/logs" "github.com/zeta-chain/zetacore/zetaclient/metrics" ) @@ -38,7 +39,10 @@ func NewSigner(chain chains.Chain, tss interfaces.TSSSigner, ts *metrics.Telemet tss: tss, ts: ts, logger: Logger{ - Std: logger.Std.With().Int64("chain", chain.ChainId).Str("module", "signer").Logger(), + Std: logger.Std.With(). + Int64(logs.FieldChain, chain.ChainId). + Str(logs.FieldModule, "signer"). + Logger(), Compliance: logger.Compliance, }, outboundBeingReported: make(map[string]bool), diff --git a/zetaclient/chains/evm/constant.go b/zetaclient/chains/evm/constant.go index b754d57f30..beaeb6143f 100644 --- a/zetaclient/chains/evm/constant.go +++ b/zetaclient/chains/evm/constant.go @@ -3,12 +3,13 @@ package evm import "time" const ( - // ZetaBlockTime is the block time of the Zeta network - ZetaBlockTime = 6500 * time.Millisecond - // OutboundInclusionTimeout is the timeout for waiting for an outbound to be included in a block OutboundInclusionTimeout = 20 * time.Minute + // ReorgProtectBlockCount is confirmations count to protect against reorg + // Short 1~2 block reorgs could happen often on Ethereum due to network congestion or block production race conditions + ReorgProtectBlockCount = 2 + // OutboundTrackerReportTimeout is the timeout for waiting for an outbound tracker report OutboundTrackerReportTimeout = 10 * time.Minute diff --git a/zetaclient/chains/evm/observer/observer.go b/zetaclient/chains/evm/observer/observer.go index 4959b13b9a..b5e7be3f6e 100644 --- a/zetaclient/chains/evm/observer/observer.go +++ b/zetaclient/chains/evm/observer/observer.go @@ -43,9 +43,6 @@ type Observer struct { // evmJSONRPC is the EVM JSON RPC client for the observed chain evmJSONRPC interfaces.EVMJSONRPCClient - // outboundPendingTransactions is the map to index pending transactions by hash - outboundPendingTransactions map[string]*ethtypes.Transaction - // outboundConfirmedReceipts is the map to index confirmed receipts by hash outboundConfirmedReceipts map[string]*ethtypes.Receipt @@ -92,7 +89,6 @@ func NewObserver( Observer: *baseObserver, evmClient: evmClient, evmJSONRPC: ethrpc.NewEthRPC(evmCfg.Endpoint), - outboundPendingTransactions: make(map[string]*ethtypes.Transaction), outboundConfirmedReceipts: make(map[string]*ethtypes.Receipt), outboundConfirmedTransactions: make(map[string]*ethtypes.Transaction), priorityFeeConfig: priorityFeeConfig{}, @@ -230,25 +226,10 @@ func (ob *Observer) WatchRPCStatus(ctx context.Context) error { } } -// SetPendingTx sets the pending transaction in memory -func (ob *Observer) SetPendingTx(nonce uint64, transaction *ethtypes.Transaction) { - ob.Mu().Lock() - defer ob.Mu().Unlock() - ob.outboundPendingTransactions[ob.OutboundID(nonce)] = transaction -} - -// GetPendingTx gets the pending transaction from memory -func (ob *Observer) GetPendingTx(nonce uint64) *ethtypes.Transaction { - ob.Mu().Lock() - defer ob.Mu().Unlock() - return ob.outboundPendingTransactions[ob.OutboundID(nonce)] -} - // SetTxNReceipt sets the receipt and transaction in memory func (ob *Observer) SetTxNReceipt(nonce uint64, receipt *ethtypes.Receipt, transaction *ethtypes.Transaction) { ob.Mu().Lock() defer ob.Mu().Unlock() - delete(ob.outboundPendingTransactions, ob.OutboundID(nonce)) // remove pending transaction, if any ob.outboundConfirmedReceipts[ob.OutboundID(nonce)] = receipt ob.outboundConfirmedTransactions[ob.OutboundID(nonce)] = transaction } diff --git a/zetaclient/chains/evm/observer/outbound.go b/zetaclient/chains/evm/observer/outbound.go index 680fb4e3af..598b9b6a44 100644 --- a/zetaclient/chains/evm/observer/outbound.go +++ b/zetaclient/chains/evm/observer/outbound.go @@ -13,25 +13,33 @@ import ( ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/pkg/errors" "github.com/rs/zerolog" - "github.com/rs/zerolog/log" "github.com/zeta-chain/protocol-contracts/pkg/contracts/evm/erc20custody.sol" "github.com/zeta-chain/protocol-contracts/pkg/contracts/evm/zetaconnector.non-eth.sol" "github.com/zeta-chain/zetacore/pkg/chains" "github.com/zeta-chain/zetacore/pkg/coin" + crosschainkeeper "github.com/zeta-chain/zetacore/x/crosschain/keeper" crosschaintypes "github.com/zeta-chain/zetacore/x/crosschain/types" "github.com/zeta-chain/zetacore/zetaclient/chains/evm" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" "github.com/zeta-chain/zetacore/zetaclient/compliance" zctx "github.com/zeta-chain/zetacore/zetaclient/context" + "github.com/zeta-chain/zetacore/zetaclient/logs" clienttypes "github.com/zeta-chain/zetacore/zetaclient/types" "github.com/zeta-chain/zetacore/zetaclient/zetacore" ) // WatchOutbound watches evm chain for outgoing txs status // TODO(revamp): move ticker function to ticker file -// TODO(revamp): move inner logic to a separate function func (ob *Observer) WatchOutbound(ctx context.Context) error { + // get app context + app, err := zctx.FromContext(ctx) + if err != nil { + return err + } + + // create outbound ticker + chainID := ob.Chain().ChainId ticker, err := clienttypes.NewDynamicTicker( fmt.Sprintf("EVM_WatchOutbound_%d", ob.Chain().ChainId), ob.GetChainParams().OutboundTicker, @@ -41,11 +49,6 @@ func (ob *Observer) WatchOutbound(ctx context.Context) error { return err } - app, err := zctx.FromContext(ctx) - if err != nil { - return err - } - ob.Logger().Outbound.Info().Msgf("WatchOutbound started for chain %d", ob.Chain().ChainId) sampledLogger := ob.Logger().Outbound.Sample(&zerolog.BasicSampler{N: 10}) defer ticker.Stop() @@ -57,38 +60,16 @@ func (ob *Observer) WatchOutbound(ctx context.Context) error { Msgf("WatchOutbound: outbound observation is disabled for chain %d", ob.Chain().ChainId) continue } - trackers, err := ob.ZetacoreClient(). - GetAllOutboundTrackerByChain(ctx, ob.Chain().ChainId, interfaces.Ascending) + + // process outbound trackers + err := ob.ProcessOutboundTrackers(ctx) if err != nil { - continue - } - for _, tracker := range trackers { - nonceInt := tracker.Nonce - if ob.IsTxConfirmed(nonceInt) { // Go to next tracker if this one already has a confirmed tx - continue - } - txCount := 0 - var outboundReceipt *ethtypes.Receipt - var outbound *ethtypes.Transaction - for _, txHash := range tracker.HashList { - if receipt, tx, ok := ob.checkConfirmedTx(ctx, txHash.TxHash, nonceInt); ok { - txCount++ - outboundReceipt = receipt - outbound = tx - ob.Logger().Outbound.Info(). - Msgf("WatchOutbound: confirmed outbound %s for chain %d nonce %d", txHash.TxHash, ob.Chain().ChainId, nonceInt) - if txCount > 1 { - ob.Logger().Outbound.Error().Msgf( - "WatchOutbound: checkConfirmedTx passed, txCount %d chain %d nonce %d receipt %v transaction %v", txCount, ob.Chain().ChainId, nonceInt, outboundReceipt, outbound) - } - } - } - if txCount == 1 { // should be only one txHash confirmed for each nonce. - ob.SetTxNReceipt(nonceInt, outboundReceipt, outbound) - } else if txCount > 1 { // should not happen. We can't tell which txHash is true. It might happen (e.g. glitchy/hacked endpoint) - ob.Logger().Outbound.Error().Msgf("WatchOutbound: confirmed multiple (%d) outbound for chain %d nonce %d", txCount, ob.Chain().ChainId, nonceInt) - } + ob.Logger(). + Outbound.Error(). + Err(err). + Msgf("WatchOutbound: error ProcessOutboundTrackers for chain %d", chainID) } + ticker.UpdateInterval(ob.GetChainParams().OutboundTicker, ob.Logger().Outbound) case <-ob.StopChannel(): ob.Logger().Outbound.Info().Msg("WatchOutbound: stopped") @@ -97,6 +78,61 @@ func (ob *Observer) WatchOutbound(ctx context.Context) error { } } +// ProcessOutboundTrackers processes outbound trackers +func (ob *Observer) ProcessOutboundTrackers(ctx context.Context) error { + chainID := ob.Chain().ChainId + trackers, err := ob.ZetacoreClient().GetAllOutboundTrackerByChain(ctx, ob.Chain().ChainId, interfaces.Ascending) + if err != nil { + return errors.Wrap(err, "GetAllOutboundTrackerByChain error") + } + + // prepare logger fields + logger := ob.Logger().Outbound.With(). + Str(logs.FieldMethod, "ProcessOutboundTrackers"). + Int64(logs.FieldChain, chainID). + Logger() + + // process outbound trackers + for _, tracker := range trackers { + // go to next tracker if this one already has a confirmed tx + nonce := tracker.Nonce + if ob.IsTxConfirmed(nonce) { + continue + } + + // check each txHash and save tx and receipt if it's legit and confirmed + txCount := 0 + var outboundReceipt *ethtypes.Receipt + var outbound *ethtypes.Transaction + for _, txHash := range tracker.HashList { + if receipt, tx, ok := ob.checkConfirmedTx(ctx, txHash.TxHash, nonce); ok { + txCount++ + outboundReceipt = receipt + outbound = tx + logger.Info().Msgf("confirmed outbound %s for chain %d nonce %d", txHash.TxHash, chainID, nonce) + if txCount > 1 { + logger.Error(). + Msgf("checkConfirmedTx passed, txCount %d chain %d nonce %d receipt %v tx %v", txCount, chainID, nonce, receipt, tx) + } + } + } + + // should be only one txHash confirmed for each nonce. + if txCount == 1 { + ob.SetTxNReceipt(nonce, outboundReceipt, outbound) + } else if txCount > 1 { + // should not happen. We can't tell which txHash is true. It might happen (e.g. bug, glitchy/hacked endpoint) + ob.Logger().Outbound.Error().Msgf("WatchOutbound: confirmed multiple (%d) outbound for chain %d nonce %d", txCount, chainID, nonce) + } else { + if len(tracker.HashList) == crosschainkeeper.MaxOutboundTrackerHashes { + ob.Logger().Outbound.Error().Msgf("WatchOutbound: outbound tracker is full of hashes for chain %d nonce %d", chainID, nonce) + } + } + } + + return nil +} + // PostVoteOutbound posts vote to zetacore for the confirmed outbound func (ob *Observer) PostVoteOutbound( ctx context.Context, @@ -377,23 +413,27 @@ func (ob *Observer) checkConfirmedTx( ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() + // prepare logger + logger := ob.Logger().Outbound.With(). + Str(logs.FieldMethod, "checkConfirmedTx"). + Int64(logs.FieldChain, ob.Chain().ChainId). + Uint64(logs.FieldNonce, nonce). + Str(logs.FieldTx, txHash). + Logger() + // query transaction transaction, isPending, err := ob.evmClient.TransactionByHash(ctx, ethcommon.HexToHash(txHash)) if err != nil { - log.Error(). - Err(err). - Str("function", "confirmTxByHash"). - Str("outboundTxHash", txHash). - Int64("chainID", ob.Chain().ChainId). - Msg("error getting transaction for outbound") + logger.Error().Err(err).Msg("TransactionByHash error") return nil, nil, false } if transaction == nil { // should not happen - log.Error(). - Str("function", "confirmTxByHash"). - Str("outboundTxHash", txHash). - Uint64("nonce", nonce). - Msg("transaction is nil for txHash") + logger.Error().Msg("transaction is nil") + return nil, nil, false + } + if isPending { + // should not happen when we are here. The outbound tracker reporter won't report a pending tx. + logger.Error().Msg("transaction is pending") return nil, nil, false } @@ -401,12 +441,7 @@ func (ob *Observer) checkConfirmedTx( signer := ethtypes.NewLondonSigner(big.NewInt(ob.Chain().ChainId)) from, err := signer.Sender(transaction) if err != nil { - log.Error(). - Err(err). - Str("function", "confirmTxByHash"). - Str("outboundTxHash", transaction.Hash().Hex()). - Int64("chainID", ob.Chain().ChainId). - Msg("local recovery of sender address failed for outbound") + logger.Error().Err(err).Msg("local recovery of sender address failed") return nil, nil, false } if from != ob.TSS().EVMAddress() { // must be TSS address @@ -416,13 +451,8 @@ func (ob *Observer) checkConfirmedTx( // TODO : improve this logic to verify that the correct TSS address is the from address. // https://github.com/zeta-chain/node/issues/2487 - log.Info(). - Str("function", "confirmTxByHash"). - Str("sender", from.Hex()). - Str("outboundTxHash", transaction.Hash().Hex()). - Int64("chainID", ob.Chain().ChainId). - Str("currentTSSAddress", ob.TSS().EVMAddress().Hex()). - Msg("sender is not current TSS address") + logger.Warn(). + Msgf("tx sender %s is not matching current TSS address %s", from.String(), ob.TSS().EVMAddress().String()) addressList := ob.TSS().EVMAddressList() isOldTssAddress := false for _, addr := range addressList { @@ -431,70 +461,35 @@ func (ob *Observer) checkConfirmedTx( } } if !isOldTssAddress { - log.Error(). - Str("function", "confirmTxByHash"). - Str("sender", from.Hex()). - Str("outboundTxHash", transaction.Hash().Hex()). - Int64("chainID", ob.Chain().ChainId). - Str("currentTSSAddress", ob.TSS().EVMAddress().Hex()). - Msg("sender is not current or old TSS address") + logger.Error().Msgf("tx sender %s is not matching any of the TSS addresses", from.String()) return nil, nil, false } } - if transaction.Nonce() != nonce { // must match cctx nonce - log.Error(). - Str("function", "confirmTxByHash"). - Str("outboundTxHash", txHash). - Uint64("wantedNonce", nonce). - Uint64("gotTxNonce", transaction.Nonce()). - Msg("outbound nonce mismatch") - return nil, nil, false - } - - // save pending transaction - if isPending { - ob.SetPendingTx(nonce, transaction) + if transaction.Nonce() != nonce { // must match tracker nonce + logger.Error().Msgf("tx nonce %d is not matching tracker nonce", nonce) return nil, nil, false } // query receipt receipt, err := ob.evmClient.TransactionReceipt(ctx, ethcommon.HexToHash(txHash)) if err != nil { - log.Error(). - Err(err). - Str("function", "confirmTxByHash"). - Str("outboundTxHash", txHash). - Uint64("nonce", nonce). - Msg("transactionReceipt error") + logger.Error().Err(err).Msg("TransactionReceipt error") return nil, nil, false } if receipt == nil { // should not happen - log.Error(). - Str("function", "confirmTxByHash"). - Str("outboundTxHash", txHash). - Uint64("nonce", nonce). - Msg("receipt is nil") + logger.Error().Msg("receipt is nil") return nil, nil, false } ob.LastBlock() // check confirmations lastHeight, err := ob.evmClient.BlockNumber(ctx) if err != nil { - log.Error(). - Str("function", "confirmTxByHash"). - Err(err). - Int64("chainID", ob.GetChainParams().ChainId). - Msg("error getting block number for chain") + logger.Error().Err(err).Msg("BlockNumber error") return nil, nil, false } if !ob.HasEnoughConfirmations(receipt, lastHeight) { - log.Debug(). - Str("function", "confirmTxByHash"). - Str("txHash", txHash). - Uint64("nonce", nonce). - Uint64("receiptBlock", receipt.BlockNumber.Uint64()). - Uint64("currentBlock", lastHeight). - Msg("txHash included but not confirmed") + logger.Debug(). + Msgf("tx included but not confirmed, receipt block %d current block %d", receipt.BlockNumber.Uint64(), lastHeight) return nil, nil, false } @@ -502,13 +497,7 @@ func (ob *Observer) checkConfirmedTx( // Note: a guard for false BlockNumber in receipt. The blob-carrying tx won't come here err = ob.CheckTxInclusion(transaction, receipt) if err != nil { - log.Error(). - Err(err). - Str("function", "confirmTxByHash"). - Str("errorContext", "checkTxInclusion"). - Str("txHash", txHash). - Uint64("nonce", nonce). - Msg("checkTxInclusion error") + logger.Error().Err(err).Msg("CheckTxInclusion error") return nil, nil, false } diff --git a/zetaclient/chains/evm/rpc/rpc.go b/zetaclient/chains/evm/rpc/rpc.go new file mode 100644 index 0000000000..6fcc3d007c --- /dev/null +++ b/zetaclient/chains/evm/rpc/rpc.go @@ -0,0 +1,52 @@ +package rpc + +import ( + "context" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/pkg/errors" + + "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" +) + +// IsTxConfirmed checks if the transaction is confirmed with given confirmations +func IsTxConfirmed( + ctx context.Context, + client interfaces.EVMRPCClient, + txHash string, + confirmations uint64, +) (bool, error) { + // query the tx + _, isPending, err := client.TransactionByHash(ctx, ethcommon.HexToHash(txHash)) + if err != nil { + return false, errors.Wrapf(err, "error getting transaction for tx %s", txHash) + } + if isPending { + return false, nil + } + + // query receipt + receipt, err := client.TransactionReceipt(ctx, ethcommon.HexToHash(txHash)) + if err != nil { + return false, errors.Wrapf(err, "error getting transaction receipt for tx %s", txHash) + } + + // should not happen + if receipt == nil { + return false, errors.Errorf("receipt is nil for tx %s", txHash) + } + + // query last block height + lastHeight, err := client.BlockNumber(ctx) + if err != nil { + return false, errors.Wrap(err, "error getting block number") + } + + // check confirmations + if lastHeight < receipt.BlockNumber.Uint64() { + return false, nil + } + blocks := lastHeight - receipt.BlockNumber.Uint64() + 1 + + return blocks >= confirmations, nil +} diff --git a/zetaclient/chains/evm/rpc/rpc_live_test.go b/zetaclient/chains/evm/rpc/rpc_live_test.go new file mode 100644 index 0000000000..0c420c830e --- /dev/null +++ b/zetaclient/chains/evm/rpc/rpc_live_test.go @@ -0,0 +1,45 @@ +package rpc_test + +import ( + "context" + "math" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/zetacore/zetaclient/chains/evm/rpc" + + "testing" +) + +const ( + URLEthMainnet = "https://rpc.ankr.com/eth" + URLEthSepolia = "https://rpc.ankr.com/eth_sepolia" + URLBscMainnet = "https://rpc.ankr.com/bsc" + URLPolygonMainnet = "https://rpc.ankr.com/polygon" +) + +// Test_EVMRPCLive is a phony test to run each live test individually +func Test_EVMRPCLive(t *testing.T) { + // LiveTest_IsTxConfirmed(t) +} + +func LiveTest_IsTxConfirmed(t *testing.T) { + client, err := ethclient.Dial(URLEthMainnet) + require.NoError(t, err) + + // check if the transaction is confirmed + ctx := context.Background() + txHash := "0xd2eba7ac3da1b62800165414ea4bcaf69a3b0fb9b13a0fc32f4be11bfef79146" + + t.Run("should confirm tx", func(t *testing.T) { + confirmed, err := rpc.IsTxConfirmed(ctx, client, txHash, 12) + require.NoError(t, err) + require.True(t, confirmed) + }) + + t.Run("should not confirm tx if confirmations is not enough", func(t *testing.T) { + confirmed, err := rpc.IsTxConfirmed(ctx, client, txHash, math.MaxUint64) + require.NoError(t, err) + require.False(t, confirmed) + }) +} diff --git a/zetaclient/chains/evm/signer/outbound_data.go b/zetaclient/chains/evm/signer/outbound_data.go index 399f4e00e0..dbeef3626d 100644 --- a/zetaclient/chains/evm/signer/outbound_data.go +++ b/zetaclient/chains/evm/signer/outbound_data.go @@ -13,7 +13,6 @@ import ( "github.com/zeta-chain/zetacore/pkg/coin" "github.com/zeta-chain/zetacore/x/crosschain/types" - "github.com/zeta-chain/zetacore/zetaclient/chains/evm/observer" zctx "github.com/zeta-chain/zetacore/zetaclient/context" ) @@ -47,7 +46,6 @@ type OutboundData struct { func NewOutboundData( ctx context.Context, cctx *types.CrossChainTx, - observer *observer.Observer, height uint64, logger zerolog.Logger, ) (*OutboundData, bool, error) { @@ -56,8 +54,6 @@ func NewOutboundData( } outboundParams := cctx.GetCurrentOutboundParam() - nonce := outboundParams.TssNonce - if err := validateParams(outboundParams); err != nil { return nil, false, errors.Wrap(err, "invalid outboundParams") } @@ -88,24 +84,6 @@ func NewOutboundData( return nil, false, errors.Wrap(err, "unable to get cctx index") } - // In case there is a pending tx, make sure this keySign is a tx replacement - if tx := observer.GetPendingTx(nonce); tx != nil { - evt := logger.Info(). - Str("cctx.pending_tx_hash", tx.Hash().Hex()). - Uint64("cctx.pending_tx_nonce", nonce) - - // new gas price is less or equal to pending tx gas - if gas.Price.Cmp(tx.GasPrice()) <= 0 { - evt.Msg("Please wait for pending outbound to be included in the block") - return nil, true, nil - } - - evt. - Uint64("cctx.gas_price", gas.Price.Uint64()). - Uint64("cctx.priority_fee", gas.PriorityFee.Uint64()). - Msg("Replacing pending outbound transaction with higher gas fees") - } - // Base64 decode message var message []byte if cctx.InboundParams.CoinType != coin.CoinType_Cmd { diff --git a/zetaclient/chains/evm/signer/outbound_data_test.go b/zetaclient/chains/evm/signer/outbound_data_test.go index 03c1329ef1..2f53bab028 100644 --- a/zetaclient/chains/evm/signer/outbound_data_test.go +++ b/zetaclient/chains/evm/signer/outbound_data_test.go @@ -13,15 +13,12 @@ import ( ) func TestNewOutboundData(t *testing.T) { - mockObserver, err := getNewEvmChainObserver(t, nil) - require.NoError(t, err) - logger := zerolog.New(zerolog.NewTestWriter(t)) ctx := makeCtx(t) newOutbound := func(cctx *types.CrossChainTx) (*OutboundData, bool, error) { - return NewOutboundData(ctx, cctx, mockObserver, 123, logger) + return NewOutboundData(ctx, cctx, 123, logger) } t.Run("success", func(t *testing.T) { diff --git a/zetaclient/chains/evm/signer/outbound_tracker_reporter.go b/zetaclient/chains/evm/signer/outbound_tracker_reporter.go new file mode 100644 index 0000000000..7a9b6bcfa2 --- /dev/null +++ b/zetaclient/chains/evm/signer/outbound_tracker_reporter.go @@ -0,0 +1,85 @@ +// Package signer implements the ChainSigner interface for EVM chains +package signer + +import ( + "context" + "time" + + "github.com/rs/zerolog" + + "github.com/zeta-chain/zetacore/pkg/bg" + "github.com/zeta-chain/zetacore/zetaclient/chains/evm" + "github.com/zeta-chain/zetacore/zetaclient/chains/evm/rpc" + "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" + "github.com/zeta-chain/zetacore/zetaclient/logs" +) + +// reportToOutboundTracker reports outboundHash to tracker only when tx receipt is available +func (signer *Signer) reportToOutboundTracker( + ctx context.Context, + zetacoreClient interfaces.ZetacoreClient, + chainID int64, + nonce uint64, + outboundHash string, + logger zerolog.Logger, +) { + // prepare logger + logger = logger.With(). + Str(logs.FieldMethod, "reportToOutboundTracker"). + Int64(logs.FieldChain, chainID). + Uint64(logs.FieldNonce, nonce). + Str(logs.FieldTx, outboundHash). + Logger() + + // set being reported flag to avoid duplicate reporting + alreadySet := signer.SetBeingReportedFlag(outboundHash) + if alreadySet { + logger.Info().Msg("outbound is being reported to tracker") + return + } + + // launch a goroutine to monitor tx confirmation status + bg.Work(ctx, func(ctx context.Context) error { + defer func() { + signer.ClearBeingReportedFlag(outboundHash) + }() + + // try monitoring tx inclusion status for 20 minutes + tStart := time.Now() + for { + // take a rest between each check + time.Sleep(10 * time.Second) + + // give up (forget about the tx) after 20 minutes of monitoring, there are 2 reasons: + // 1. the gas stability pool should have kicked in and replaced the tx by then. + // 2. even if there is a chance that the tx is included later, most likely it's going to be a false tx hash (either replaced or dropped). + // 3. we prefer missed tx hash over potentially invalid txhash. + if time.Since(tStart) > evm.OutboundInclusionTimeout { + logger.Info().Msgf("timeout waiting outbound inclusion") + return nil + } + + // check tx confirmation status + confirmed, err := rpc.IsTxConfirmed(ctx, signer.client, outboundHash, evm.ReorgProtectBlockCount) + if err != nil { + logger.Err(err).Msg("unable to check confirmation status of outbound") + continue + } + if !confirmed { + continue + } + + // report outbound hash to tracker + zetaHash, err := zetacoreClient.AddOutboundTracker(ctx, chainID, nonce, outboundHash, nil, "", -1) + if err != nil { + logger.Err(err).Msg("error adding outbound to tracker") + } else if zetaHash != "" { + logger.Info().Msgf("added outbound to tracker; zeta txhash %s", zetaHash) + } else { + // exit goroutine until the tracker contains the hash (reported by either this or other signers) + logger.Info().Msg("outbound now exists in tracker") + return nil + } + } + }, bg.WithName("TrackerReporterEVM"), bg.WithLogger(logger)) +} diff --git a/zetaclient/chains/evm/signer/signer.go b/zetaclient/chains/evm/signer/signer.go index 2f7737fc8a..874c118e3e 100644 --- a/zetaclient/chains/evm/signer/signer.go +++ b/zetaclient/chains/evm/signer/signer.go @@ -20,18 +20,16 @@ import ( ethrpc "github.com/ethereum/go-ethereum/rpc" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/zeta-chain/protocol-contracts/pkg/contracts/evm/erc20custody.sol" "github.com/zeta-chain/zetacore/pkg/chains" "github.com/zeta-chain/zetacore/pkg/coin" - crosschainkeeper "github.com/zeta-chain/zetacore/x/crosschain/keeper" "github.com/zeta-chain/zetacore/x/crosschain/types" "github.com/zeta-chain/zetacore/zetaclient/chains/base" "github.com/zeta-chain/zetacore/zetaclient/chains/evm" - "github.com/zeta-chain/zetacore/zetaclient/chains/evm/observer" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" "github.com/zeta-chain/zetacore/zetaclient/compliance" zctx "github.com/zeta-chain/zetacore/zetaclient/context" + "github.com/zeta-chain/zetacore/zetaclient/logs" "github.com/zeta-chain/zetacore/zetaclient/metrics" "github.com/zeta-chain/zetacore/zetaclient/outboundprocessor" "github.com/zeta-chain/zetacore/zetaclient/testutils/mocks" @@ -376,48 +374,42 @@ func (signer *Signer) TryProcessOutbound( cctx *types.CrossChainTx, outboundProc *outboundprocessor.Processor, outboundID string, - chainObserver interfaces.ChainObserver, + _ interfaces.ChainObserver, zetacoreClient interfaces.ZetacoreClient, height uint64, ) { + // end outbound process on panic + defer func() { + outboundProc.EndTryProcess(outboundID) + if r := recover(); r != nil { + signer.Logger().Std.Error().Msgf("TryProcessOutbound: %s, caught panic error: %v", cctx.Index, r) + } + }() + + // prepare logger and a few local variables var ( params = cctx.GetCurrentOutboundParam() myID = zetacoreClient.GetKeys().GetOperatorAddress() logger = signer.Logger().Std.With(). - Str("method", "TryProcessOutbound"). - Int64("chain", signer.Chain().ChainId). - Uint64("nonce", params.TssNonce). - Str("cctx.index", cctx.Index). + Str(logs.FieldMethod, "TryProcessOutbound"). + Int64(logs.FieldChain, signer.Chain().ChainId). + Uint64(logs.FieldNonce, params.TssNonce). + Str(logs.FieldCctx, cctx.Index). Str("cctx.receiver", params.Receiver). Str("cctx.amount", params.Amount.String()). Logger() ) - logger.Info().Msgf("TryProcessOutbound") + // retrieve app context app, err := zctx.FromContext(ctx) if err != nil { - signer.Logger().Std.Error().Err(err).Msg("error getting app context") - return - } - - // end outbound process on panic - defer func() { - if r := recover(); r != nil { - logger.Error().Interface("panic", r).Msg("panic in TryProcessOutbound") - } - - outboundProc.EndTryProcess(outboundID) - }() - - evmObserver, ok := chainObserver.(*observer.Observer) - if !ok { - logger.Error().Msg("chain observer is not an EVM observer") + logger.Error().Err(err).Msg("error getting app context") return } // Setup Transaction input - txData, skipTx, err := NewOutboundData(ctx, cctx, evmObserver, height, logger) + txData, skipTx, err := NewOutboundData(ctx, cctx, height, logger) if err != nil { logger.Err(err).Msg("error setting up transaction input fields") return @@ -713,172 +705,6 @@ func ErrorMsg(cctx *types.CrossChainTx) string { ) } -// SignWhitelistERC20Cmd signs a whitelist command for ERC20 token -// TODO(revamp): move the cmd in a specific file -func (signer *Signer) SignWhitelistERC20Cmd( - ctx context.Context, - txData *OutboundData, - params string, -) (*ethtypes.Transaction, error) { - outboundParams := txData.outboundParams - erc20 := ethcommon.HexToAddress(params) - if erc20 == (ethcommon.Address{}) { - return nil, fmt.Errorf("SignCommandTx: invalid erc20 address %s", params) - } - - custodyAbi, err := erc20custody.ERC20CustodyMetaData.GetAbi() - if err != nil { - return nil, err - } - - data, err := custodyAbi.Pack("whitelist", erc20) - if err != nil { - return nil, fmt.Errorf("whitelist pack error: %w", err) - } - - tx, _, _, err := signer.Sign( - ctx, - data, - txData.to, - zeroValue, - txData.gas, - outboundParams.TssNonce, - txData.height, - ) - if err != nil { - return nil, fmt.Errorf("sign whitelist error: %w", err) - } - - return tx, nil -} - -// SignMigrateTssFundsCmd signs a migrate TSS funds command -// TODO(revamp): move the cmd in a specific file -func (signer *Signer) SignMigrateTssFundsCmd(ctx context.Context, txData *OutboundData) (*ethtypes.Transaction, error) { - tx, _, _, err := signer.Sign( - ctx, - nil, - txData.to, - txData.amount, - txData.gas, - txData.nonce, - txData.height, - ) - if err != nil { - return nil, fmt.Errorf("SignMigrateTssFundsCmd error: %w", err) - } - return tx, nil -} - -// reportToOutboundTracker reports outboundHash to tracker only when tx receipt is available -// TODO(revamp): move outbound tracker function to a outbound tracker file -func (signer *Signer) reportToOutboundTracker( - ctx context.Context, - zetacoreClient interfaces.ZetacoreClient, - chainID int64, - nonce uint64, - outboundHash string, - logger zerolog.Logger, -) { - // set being reported flag to avoid duplicate reporting - alreadySet := signer.Signer.SetBeingReportedFlag(outboundHash) - if alreadySet { - logger.Info(). - Msgf("reportToOutboundTracker: outboundHash %s for chain %d nonce %d is being reported", outboundHash, chainID, nonce) - return - } - - // report to outbound tracker with goroutine - go func() { - defer func() { - signer.Signer.ClearBeingReportedFlag(outboundHash) - }() - - // try monitoring tx inclusion status for 10 minutes - var err error - report := false - isPending := false - blockNumber := uint64(0) - tStart := time.Now() - for { - // give up after 10 minutes of monitoring - time.Sleep(10 * time.Second) - - if time.Since(tStart) > evm.OutboundInclusionTimeout { - // if tx is still pending after timeout, report to outboundTracker anyway as we cannot monitor forever - if isPending { - report = true // probably will be included later - } - logger.Info(). - Msgf("reportToOutboundTracker: timeout waiting tx inclusion for chain %d nonce %d outboundHash %s report %v", chainID, nonce, outboundHash, report) - break - } - // try getting the tx - _, isPending, err = signer.client.TransactionByHash(ctx, ethcommon.HexToHash(outboundHash)) - if err != nil { - logger.Info(). - Err(err). - Msgf("reportToOutboundTracker: error getting tx for chain %d nonce %d outboundHash %s", chainID, nonce, outboundHash) - continue - } - // if tx is include in a block, try getting receipt - if !isPending { - report = true // included - receipt, err := signer.client.TransactionReceipt(ctx, ethcommon.HexToHash(outboundHash)) - if err != nil { - logger.Info(). - Err(err). - Msgf("reportToOutboundTracker: error getting receipt for chain %d nonce %d outboundHash %s", chainID, nonce, outboundHash) - } - if receipt != nil { - blockNumber = receipt.BlockNumber.Uint64() - } - break - } - // keep monitoring pending tx - logger.Info(). - Msgf("reportToOutboundTracker: tx has not been included yet for chain %d nonce %d outboundHash %s", chainID, nonce, outboundHash) - } - - // try adding to outbound tracker for 10 minutes - if report { - tStart := time.Now() - for { - // give up after 10 minutes of retrying - if time.Since(tStart) > evm.OutboundTrackerReportTimeout { - logger.Info(). - Msgf("reportToOutboundTracker: timeout adding outbound tracker for chain %d nonce %d outboundHash %s, please add manually", chainID, nonce, outboundHash) - break - } - // stop if the cctx is already finalized - cctx, err := zetacoreClient.GetCctxByNonce(ctx, chainID, nonce) - if err != nil { - logger.Err(err). - Msgf("reportToOutboundTracker: error getting cctx for chain %d nonce %d outboundHash %s", chainID, nonce, outboundHash) - } else if !crosschainkeeper.IsPending(cctx) { - logger.Info().Msgf("reportToOutboundTracker: cctx already finalized for chain %d nonce %d outboundHash %s", chainID, nonce, outboundHash) - break - } - // report to outbound tracker - zetaHash, err := zetacoreClient.AddOutboundTracker(ctx, chainID, nonce, outboundHash, nil, "", -1) - if err != nil { - logger.Err(err). - Msgf("reportToOutboundTracker: error adding to outbound tracker for chain %d nonce %d outboundHash %s", chainID, nonce, outboundHash) - } else if zetaHash != "" { - logger.Info().Msgf("reportToOutboundTracker: added outboundHash to core successful %s, chain %d nonce %d outboundHash %s block %d", - zetaHash, chainID, nonce, outboundHash, blockNumber) - } else { - // stop if the tracker contains the outboundHash - logger.Info().Msgf("reportToOutboundTracker: outbound tracker contains outboundHash %s for chain %d nonce %d", outboundHash, chainID, nonce) - break - } - // retry otherwise - time.Sleep(evm.ZetaBlockTime * 3) - } - } - }() -} - // getEVMRPC is a helper function to set up the client and signer, also initializes a mock client for unit tests func getEVMRPC(ctx context.Context, endpoint string) (interfaces.EVMRPCClient, ethtypes.Signer, error) { if endpoint == mocks.EVMRPCEnabled { diff --git a/zetaclient/chains/evm/signer/signer_admin_test.go b/zetaclient/chains/evm/signer/signer_admin_test.go index 353fd360ca..820b521892 100644 --- a/zetaclient/chains/evm/signer/signer_admin_test.go +++ b/zetaclient/chains/evm/signer/signer_admin_test.go @@ -20,9 +20,8 @@ func TestSigner_SignAdminTx(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, nil) require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.False(t, skip) require.NoError(t, err) @@ -89,10 +88,9 @@ func TestSigner_SignWhitelistERC20Cmd(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.NoError(t, err) require.False(t, skip) @@ -138,10 +136,9 @@ func TestSigner_SignMigrateERC20CustodyFundsCmd(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.NoError(t, err) require.False(t, skip) @@ -203,9 +200,8 @@ func TestSigner_SignMigrateTssFundsCmd(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.False(t, skip) require.NoError(t, err) diff --git a/zetaclient/chains/evm/signer/signer_test.go b/zetaclient/chains/evm/signer/signer_test.go index 89eeeef740..570cd57042 100644 --- a/zetaclient/chains/evm/signer/signer_test.go +++ b/zetaclient/chains/evm/signer/signer_test.go @@ -195,9 +195,7 @@ func TestSigner_SignOutbound(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) - require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, makeLogger(t)) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.False(t, skip) require.NoError(t, err) @@ -237,7 +235,7 @@ func TestSigner_SignOutbound(t *testing.T) { cctx.OutboundParams[0].GasPriorityFee = big.NewInt(priorityFee).String() // Given outbound data - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, makeLogger(t)) + txData, skip, err := NewOutboundData(ctx, cctx, 123, makeLogger(t)) require.False(t, skip) require.NoError(t, err) @@ -270,9 +268,7 @@ func TestSigner_SignRevertTx(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) - require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.False(t, skip) require.NoError(t, err) @@ -310,9 +306,7 @@ func TestSigner_SignCancelTx(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) - require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.False(t, skip) require.NoError(t, err) @@ -350,9 +344,7 @@ func TestSigner_SignWithdrawTx(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) - require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.False(t, skip) require.NoError(t, err) @@ -389,9 +381,7 @@ func TestSigner_SignERC20WithdrawTx(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, tss) - require.NoError(t, err) - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.False(t, skip) require.NoError(t, err) @@ -429,10 +419,7 @@ func TestSigner_BroadcastOutbound(t *testing.T) { // Setup txData struct cctx := getCCTX(t) - mockObserver, err := getNewEvmChainObserver(t, nil) - require.NoError(t, err) - - txData, skip, err := NewOutboundData(ctx, cctx, mockObserver, 123, zerolog.Logger{}) + txData, skip, err := NewOutboundData(ctx, cctx, 123, zerolog.Logger{}) require.NoError(t, err) require.False(t, skip) diff --git a/zetaclient/chains/solana/observer/outbound.go b/zetaclient/chains/solana/observer/outbound.go index 6eca0b437e..396e49df1c 100644 --- a/zetaclient/chains/solana/observer/outbound.go +++ b/zetaclient/chains/solana/observer/outbound.go @@ -17,6 +17,7 @@ import ( crosschaintypes "github.com/zeta-chain/zetacore/x/crosschain/types" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" zctx "github.com/zeta-chain/zetacore/zetaclient/context" + "github.com/zeta-chain/zetacore/zetaclient/logs" clienttypes "github.com/zeta-chain/zetacore/zetaclient/types" "github.com/zeta-chain/zetacore/zetaclient/zetacore" ) @@ -254,15 +255,15 @@ func (ob *Observer) CheckFinalizedTx( // prepare logger fields chainID := ob.Chain().ChainId logger := ob.Logger().Outbound.With(). - Str("method", "checkFinalizedTx"). - Int64("chain", chainID). - Uint64("nonce", nonce). - Str("tx", txHash).Logger() + Str(logs.FieldMethod, "CheckFinalizedTx"). + Int64(logs.FieldChain, chainID). + Uint64(logs.FieldNonce, nonce). + Str(logs.FieldTx, txHash).Logger() // convert txHash to signature sig, err := solana.SignatureFromBase58(txHash) if err != nil { - logger.Error().Err(err).Msgf("SignatureFromBase58 err for chain %d nonce %d", chainID, nonce) + logger.Error().Err(err).Msg("SignatureFromBase58 error") return nil, false } @@ -271,20 +272,20 @@ func (ob *Observer) CheckFinalizedTx( Commitment: rpc.CommitmentFinalized, }) if err != nil { - logger.Error().Err(err).Msgf("GetTransaction err for chain %d nonce %d", chainID, nonce) + logger.Error().Err(err).Msg("GetTransaction error") return nil, false } // the tx must be successful in order to effectively increment the nonce if txResult.Meta.Err != nil { - logger.Error().Any("Err", txResult.Meta.Err).Msgf("tx is not successful for chain %d nonce %d", chainID, nonce) + logger.Error().Any("Err", txResult.Meta.Err).Msg("tx is not successful") return nil, false } // parse gateway instruction from tx result inst, err := ParseGatewayInstruction(txResult, ob.gatewayID, coinType) if err != nil { - logger.Error().Err(err).Msgf("ParseGatewayInstruction err for chain %d nonce %d", chainID, nonce) + logger.Error().Err(err).Msg("ParseGatewayInstruction error") return nil, false } txNonce := inst.GatewayNonce() @@ -292,19 +293,19 @@ func (ob *Observer) CheckFinalizedTx( // recover ECDSA signer from instruction signerECDSA, err := inst.Signer() if err != nil { - logger.Error().Err(err).Msgf("cannot get instruction signer for chain %d nonce %d", chainID, nonce) + logger.Error().Err(err).Msg("cannot get instruction signer") return nil, false } // check tx authorization if signerECDSA != ob.TSS().EVMAddress() { - logger.Error().Msgf("tx signer %s is not matching TSS, chain %d nonce %d", signerECDSA, chainID, nonce) + logger.Error().Msgf("tx signer %s is not matching current TSS address %s", signerECDSA, ob.TSS().EVMAddress()) return nil, false } // check tx nonce if txNonce != nonce { - logger.Error().Msgf("tx nonce %d is not matching cctx, chain %d nonce %d", txNonce, chainID, nonce) + logger.Error().Msgf("tx nonce %d is not matching tracker nonce", txNonce) return nil, false } diff --git a/zetaclient/chains/solana/signer/outbound_tracker_reporter.go b/zetaclient/chains/solana/signer/outbound_tracker_reporter.go index 6462060beb..0a5fb6432e 100644 --- a/zetaclient/chains/solana/signer/outbound_tracker_reporter.go +++ b/zetaclient/chains/solana/signer/outbound_tracker_reporter.go @@ -8,7 +8,9 @@ import ( "github.com/gagliardetto/solana-go/rpc" "github.com/rs/zerolog" + "github.com/zeta-chain/zetacore/pkg/bg" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" + "github.com/zeta-chain/zetacore/zetaclient/logs" ) const ( @@ -27,16 +29,23 @@ func (signer *Signer) reportToOutboundTracker( txSig solana.Signature, logger zerolog.Logger, ) { + // prepare logger + logger = logger.With(). + Str(logs.FieldMethod, "reportToOutboundTracker"). + Int64(logs.FieldChain, chainID). + Uint64(logs.FieldNonce, nonce). + Str(logs.FieldTx, txSig.String()). + Logger() + // set being reported flag to avoid duplicate reporting alreadySet := signer.Signer.SetBeingReportedFlag(txSig.String()) if alreadySet { - logger.Info(). - Msgf("reportToOutboundTracker: outbound %s for chain %d nonce %d is being reported", txSig, chainID, nonce) + logger.Info().Msg("outbound is being reported to tracker") return } // launch a goroutine to monitor tx confirmation status - go func() { + bg.Work(ctx, func(ctx context.Context) error { defer func() { signer.Signer.ClearBeingReportedFlag(txSig.String()) }() @@ -48,9 +57,8 @@ func (signer *Signer) reportToOutboundTracker( // give up if we know the tx is too old and already expired if time.Since(start) > SolanaTransactionTimeout { - logger.Info(). - Msgf("reportToOutboundTracker: outbound %s expired for chain %d nonce %d", txSig, chainID, nonce) - return + logger.Info().Msg("outbound is expired") + return nil } // query tx using optimistic commitment level "confirmed" @@ -68,24 +76,21 @@ func (signer *Signer) reportToOutboundTracker( // unlike Ethereum, Solana doesn't have protocol-level nonce; the nonce is enforced by the gateway program. // a failed outbound (e.g. signature err, balance err) will never be able to increment the gateway program nonce. // a good/valid candidate of outbound tracker hash must come with a successful tx. - logger.Warn(). - Any("Err", tx.Meta.Err). - Msgf("reportToOutboundTracker: outbound %s failed for chain %d nonce %d", txSig, chainID, nonce) - return + logger.Warn().Any("Err", tx.Meta.Err).Msg("outbound is failed") + return nil } // report outbound hash to zetacore zetaHash, err := zetacoreClient.AddOutboundTracker(ctx, chainID, nonce, txSig.String(), nil, "", -1) if err != nil { - logger.Err(err). - Msgf("reportToOutboundTracker: error adding outbound %s for chain %d nonce %d", txSig, chainID, nonce) + logger.Err(err).Msg("error adding outbound to tracker") } else if zetaHash != "" { - logger.Info().Msgf("reportToOutboundTracker: added outbound %s for chain %d nonce %d; zeta txhash %s", txSig, chainID, nonce, zetaHash) + logger.Info().Msgf("added outbound to tracker; zeta txhash %s", zetaHash) } else { - // exit goroutine if the tracker already contains the hash (reported by other signer) - logger.Info().Msgf("reportToOutboundTracker: outbound %s already in tracker for chain %d nonce %d", txSig, chainID, nonce) - return + // exit goroutine until the tracker contains the hash (reported by either this or other signers) + logger.Info().Msg("outbound now exists in tracker") + return nil } } - }() + }, bg.WithName("TrackerReporterSolana"), bg.WithLogger(logger)) } diff --git a/zetaclient/logs/fields.go b/zetaclient/logs/fields.go new file mode 100644 index 0000000000..497690ffa4 --- /dev/null +++ b/zetaclient/logs/fields.go @@ -0,0 +1,18 @@ +package logs + +// A group of predefined field keys and module names for zetaclient logs +const ( + // field keys + FieldModule = "module" + FieldMethod = "method" + FieldChain = "chain" + FieldNonce = "nonce" + FieldTx = "tx" + FieldCctx = "cctx" + + // module names + ModNameInbound = "inbound" + ModNameOutbound = "outbound" + ModNameGasPrice = "gasprice" + ModNameHeaders = "headers" +) diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index 884cbdc485..a7f8feccfb 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -15,12 +15,12 @@ import ( "github.com/samber/lo" "github.com/zeta-chain/zetacore/pkg/bg" + "github.com/zeta-chain/zetacore/pkg/constant" zetamath "github.com/zeta-chain/zetacore/pkg/math" "github.com/zeta-chain/zetacore/x/crosschain/types" observertypes "github.com/zeta-chain/zetacore/x/observer/types" "github.com/zeta-chain/zetacore/zetaclient/chains/base" btcobserver "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/observer" - "github.com/zeta-chain/zetacore/zetaclient/chains/evm" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" solanaobserver "github.com/zeta-chain/zetacore/zetaclient/chains/solana/observer" zctx "github.com/zeta-chain/zetacore/zetaclient/context" @@ -660,8 +660,13 @@ func (oc *Orchestrator) ScheduleCctxSolana( // runObserverSignerSync runs a blocking ticker that observes chain changes from zetacore // and optionally (de)provisions respective observers and signers. func (oc *Orchestrator) runObserverSignerSync(ctx context.Context) error { - // check every other zeta block - const cadence = 2 * evm.ZetaBlockTime + // sync observers and signers right away to speed up zetaclient startup + if err := oc.syncObserverSigner(ctx); err != nil { + oc.logger.Error().Err(err).Msg("runObserverSignerSync: syncObserverSigner failed for initial sync") + } + + // sync observer and signer every 10 blocks (approx. 1 minute) + const cadence = 10 * constant.ZetaBlockTime ticker := time.NewTicker(cadence) defer ticker.Stop()