diff --git a/zetaclient/chains/base/observer.go b/zetaclient/chains/base/observer.go index 1778e2d202..a948ac396f 100644 --- a/zetaclient/chains/base/observer.go +++ b/zetaclient/chains/base/observer.go @@ -7,6 +7,7 @@ import ( "strconv" "sync" "sync/atomic" + "time" lru "github.com/hashicorp/golang-lru" "github.com/pkg/errors" @@ -60,8 +61,8 @@ type Observer struct { // lastTxScanned is the last transaction hash scanned by the observer lastTxScanned string - // rpcAlertLatency is the threshold of RPC latency (in seconds) to trigger an alert - rpcAlertLatency uint64 + // rpcAlertLatency is the threshold of RPC latency to trigger an alert + rpcAlertLatency time.Duration // blockCache is the cache for blocks blockCache *lru.Cache @@ -95,7 +96,7 @@ func NewObserver( tss interfaces.TSSSigner, blockCacheSize int, headerCacheSize int, - rpcAlertLatency uint64, + rpcAlertLatency time.Duration, ts *metrics.TelemetryServer, database *db.DB, logger Logger, @@ -108,7 +109,7 @@ func NewObserver( lastBlock: 0, lastBlockScanned: 0, lastTxScanned: "", - rpcAlertLatency: rpcAlertLatency, + rpcAlertLatency: rpcAlertLatency * time.Second, // latency in seconds ts: ts, db: database, mu: &sync.Mutex{}, @@ -252,7 +253,7 @@ func (ob *Observer) WithLastTxScanned(txHash string) *Observer { } // RPCAlertLatency returns the RPC alert latency for the observer. -func (ob *Observer) RPCAlertLatency() uint64 { +func (ob *Observer) RPCAlertLatency() time.Duration { return ob.rpcAlertLatency } diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 65adf2f932..7a74164fff 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -8,6 +8,7 @@ import ( "math" "math/big" "sort" + "time" "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/chaincfg" @@ -114,7 +115,7 @@ func NewObserver( chainParams observertypes.ChainParams, zetacoreClient interfaces.ZetacoreClient, tss interfaces.TSSSigner, - rpcAlertLatency uint64, + rpcAlertLatency time.Duration, database *db.DB, logger base.Logger, ts *metrics.TelemetryServer, diff --git a/zetaclient/chains/bitcoin/rpc/rpc.go b/zetaclient/chains/bitcoin/rpc/rpc.go index d422ba66cb..c2f73364d8 100644 --- a/zetaclient/chains/bitcoin/rpc/rpc.go +++ b/zetaclient/chains/bitcoin/rpc/rpc.go @@ -26,7 +26,7 @@ const ( // RPCAlertLatency is the default threshold for RPC latency to be considered unhealthy and trigger an alert. // Bitcoin block time is 10 minutes, 1200s (20 minutes) is a reasonable threshold for Bitcoin - RPCAlertLatency = 1200 + RPCAlertLatency = time.Duration(1200) * time.Second ) // NewRPCClient creates a new RPC client by the given config. @@ -168,7 +168,7 @@ func GetRecentFeeRate(rpcClient interfaces.BTCRPCClient, netParams *chaincfg.Par // CheckRPCStatus checks the RPC status of the evm chain func CheckRPCStatus( client interfaces.BTCRPCClient, - alertLatency uint64, + alertLatency time.Duration, tssAddress btcutil.Address, logger zerolog.Logger, ) error { @@ -191,18 +191,18 @@ func CheckRPCStatus( } // use default alert latency if not provided - if alertLatency == 0 { + if alertLatency <= 0 { alertLatency = RPCAlertLatency } // latest block should not be too old blockTime := header.Timestamp - elapsedSeconds := time.Since(blockTime).Seconds() - if elapsedSeconds > float64(alertLatency) { + elapsedTime := time.Since(blockTime) + if elapsedTime > alertLatency { return errors.Errorf( "Latest block %d is %.0fs old, RPC stale or chain stuck (check explorer)?", bn, - elapsedSeconds, + elapsedTime.Seconds(), ) } @@ -218,6 +218,6 @@ func CheckRPCStatus( } logger.Info(). - Msgf("RPC Status [OK]: latest block %d, timestamp %s (%.fs ago), tss addr %s, #utxos: %d", bn, blockTime, elapsedSeconds, tssAddress, len(res)) + Msgf("RPC Status [OK]: latest block %d, timestamp %s (%.fs ago), tss addr %s, #utxos: %d", bn, blockTime, elapsedTime.Seconds(), tssAddress, len(res)) return nil } diff --git a/zetaclient/chains/bitcoin/rpc/rpc_live_test.go b/zetaclient/chains/bitcoin/rpc/rpc_live_test.go index ebc8bf1a50..cde5fae2b7 100644 --- a/zetaclient/chains/bitcoin/rpc/rpc_live_test.go +++ b/zetaclient/chains/bitcoin/rpc/rpc_live_test.go @@ -26,6 +26,7 @@ import ( "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin" "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/observer" "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/rpc" + "github.com/zeta-chain/zetacore/zetaclient/common" "github.com/zeta-chain/zetacore/zetaclient/config" "github.com/zeta-chain/zetacore/zetaclient/testutils" "github.com/zeta-chain/zetacore/zetaclient/testutils/mocks" @@ -98,8 +99,8 @@ func (suite *BitcoinObserverTestSuite) TearDownSuite() { // createRPCClient creates a new Bitcoin RPC client for given chainID func createRPCClient(chainID int64) (*rpcclient.Client, error) { var connCfg *rpcclient.ConnConfig - rpcMainnet := os.Getenv("BTC_RPC_MAINNET") - rpcTestnet := os.Getenv("BTC_RPC_TESTNET") + rpcMainnet := os.Getenv(common.EnvBtcRPCMainnet) + rpcTestnet := os.Getenv(common.EnvBtcRPCTestnet) // mainnet if chainID == chains.BitcoinMainnet.ChainId { @@ -216,16 +217,21 @@ func (suite *BitcoinObserverTestSuite) Test2() { // TestBitcoinObserverLive is a phony test to run each live test individually func TestBitcoinObserverLive(t *testing.T) { + if !common.LiveTestEnabled() { + return + } + + // disable legacy live tests // suite.Run(t, new(BitcoinClientTestSuite)) - // LiveTestNewRPCClient(t) - // LiveTestCheckRPCStatus(t) - // LiveTestGetBlockHeightByHash(t) - // LiveTestBitcoinFeeRate(t) - // LiveTestAvgFeeRateMainnetMempoolSpace(t) - // LiveTestAvgFeeRateTestnetMempoolSpace(t) - // LiveTestGetRecentFeeRate(t) - // LiveTestGetSenderByVin(t) + LiveTestNewRPCClient(t) + LiveTestCheckRPCStatus(t) + LiveTestGetBlockHeightByHash(t) + LiveTestBitcoinFeeRate(t) + LiveTestAvgFeeRateMainnetMempoolSpace(t) + LiveTestAvgFeeRateTestnetMempoolSpace(t) + LiveTestGetRecentFeeRate(t) + LiveTestGetSenderByVin(t) } // LiveTestNewRPCClient creates a new Bitcoin RPC client @@ -233,8 +239,8 @@ func LiveTestNewRPCClient(t *testing.T) { btcConfig := config.BTCConfig{ RPCUsername: "user", RPCPassword: "pass", - RPCHost: os.Getenv("BTC_RPC_TESTNET"), - RPCParams: "mainnet", + RPCHost: os.Getenv(common.EnvBtcRPCTestnet), + RPCParams: "testnet3", } // create Bitcoin RPC client @@ -326,9 +332,11 @@ func LiveTestBitcoinFeeRate(t *testing.T) { feeRateEconomical2.Uint64(), ) - // monitor fee rate every 5 minutes - for { - time.Sleep(time.Duration(5) * time.Minute) + // monitor fee rate every 5 minutes, adjust the iteration count as needed + for i := 0; i < 1; i++ { + // please uncomment this interval for long running test + //time.Sleep(time.Duration(5) * time.Minute) + bn, err = client.GetBlockCount() feeRateConservative1, errCon1 = getFeeRate(client, 1, &btcjson.EstimateModeConservative) feeRateEconomical1, errEco1 = getFeeRate(client, 1, &btcjson.EstimateModeEconomical) @@ -419,7 +427,7 @@ func LiveTestAvgFeeRateMainnetMempoolSpace(t *testing.T) { // test against mempool.space API for 10000 blocks //startBlock := 210000 * 3 // 3rd halving startBlock := 829596 - endBlock := startBlock - 10000 + endBlock := startBlock - 1 // go back to whatever block as needed compareAvgFeeRate(t, client, startBlock, endBlock, false) } @@ -433,7 +441,7 @@ func LiveTestAvgFeeRateTestnetMempoolSpace(t *testing.T) { // test against mempool.space API for 10000 blocks //startBlock := 210000 * 12 // 12th halving startBlock := 2577600 - endBlock := startBlock - 10000 + endBlock := startBlock - 1 // go back to whatever block as needed compareAvgFeeRate(t, client, startBlock, endBlock, true) } @@ -468,7 +476,7 @@ func LiveTestGetSenderByVin(t *testing.T) { // calculates block range to test startBlock, err := client.GetBlockCount() require.NoError(t, err) - endBlock := startBlock - 5000 + endBlock := startBlock - 1 // go back to whatever block as needed // loop through mempool.space blocks in descending order BLOCKLOOP: diff --git a/zetaclient/chains/evm/observer/observer.go b/zetaclient/chains/evm/observer/observer.go index fdc6efbe61..5ed0acf136 100644 --- a/zetaclient/chains/evm/observer/observer.go +++ b/zetaclient/chains/evm/observer/observer.go @@ -7,6 +7,7 @@ import ( "math" "math/big" "strings" + "time" ethcommon "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" @@ -77,7 +78,7 @@ func NewObserver( tss, base.DefaultBlockCacheSize, base.DefaultHeaderCacheSize, - evmCfg.RPCAlertLatency, + time.Duration(evmCfg.RPCAlertLatency), ts, database, logger, diff --git a/zetaclient/chains/evm/rpc/rpc.go b/zetaclient/chains/evm/rpc/rpc.go index 6c9cb264d2..538392adae 100644 --- a/zetaclient/chains/evm/rpc/rpc.go +++ b/zetaclient/chains/evm/rpc/rpc.go @@ -15,7 +15,7 @@ import ( const ( // RPCAlertLatency is the default threshold for RPC latency to be considered unhealthy and trigger an alert. // 100s is a reasonable threshold for most EVM chains - RPCAlertLatency = 100 + RPCAlertLatency = time.Duration(100) * time.Second ) // IsTxConfirmed checks if the transaction is confirmed with given confirmations @@ -64,7 +64,7 @@ func IsTxConfirmed( func CheckRPCStatus( ctx context.Context, client interfaces.EVMRPCClient, - alertLatency uint64, + alertLatency time.Duration, logger zerolog.Logger, ) error { // query latest block number @@ -86,23 +86,23 @@ func CheckRPCStatus( } // use default alert latency if not provided - if alertLatency == 0 { + if alertLatency <= 0 { alertLatency = RPCAlertLatency } // latest block should not be too old // #nosec G115 always in range blockTime := time.Unix(int64(header.Time), 0).UTC() - elapsedSeconds := time.Since(blockTime).Seconds() - if elapsedSeconds > float64(alertLatency) { + elapsedTime := time.Since(blockTime) + if elapsedTime > alertLatency { return errors.Errorf( "Latest block %d is %.0fs old, RPC stale or chain stuck (check explorer)?", bn, - elapsedSeconds, + elapsedTime.Seconds(), ) } logger.Info(). - Msgf("RPC Status [OK]: latest block %d, timestamp %s (%.0fs ago), gas price %s", header.Number, blockTime.String(), elapsedSeconds, gasPrice.String()) + Msgf("RPC Status [OK]: latest block %d, timestamp %s (%.0fs ago), gas price %s", header.Number, blockTime.String(), elapsedTime.Seconds(), gasPrice.String()) return nil } diff --git a/zetaclient/chains/evm/rpc/rpc_live_test.go b/zetaclient/chains/evm/rpc/rpc_live_test.go index 6ba0f3bbf7..c6f2ad2368 100644 --- a/zetaclient/chains/evm/rpc/rpc_live_test.go +++ b/zetaclient/chains/evm/rpc/rpc_live_test.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog/log" "github.com/stretchr/testify/require" "github.com/zeta-chain/zetacore/zetaclient/chains/evm/rpc" + "github.com/zeta-chain/zetacore/zetaclient/common" "testing" ) @@ -21,8 +22,12 @@ const ( // Test_EVMRPCLive is a phony test to run each live test individually func Test_EVMRPCLive(t *testing.T) { - // LiveTest_IsTxConfirmed(t) - // LiveTest_CheckRPCStatus(t) + if !common.LiveTestEnabled() { + return + } + + LiveTest_IsTxConfirmed(t) + LiveTest_CheckRPCStatus(t) } func LiveTest_IsTxConfirmed(t *testing.T) { diff --git a/zetaclient/chains/solana/observer/observer.go b/zetaclient/chains/solana/observer/observer.go index 0de212b30b..cc2e3a8204 100644 --- a/zetaclient/chains/solana/observer/observer.go +++ b/zetaclient/chains/solana/observer/observer.go @@ -2,6 +2,7 @@ package observer import ( "context" + "time" "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/rpc" @@ -44,7 +45,7 @@ func NewObserver( chainParams observertypes.ChainParams, zetacoreClient interfaces.ZetacoreClient, tss interfaces.TSSSigner, - rpcAlertLatency uint64, + rpcAlertLatency time.Duration, db *db.DB, logger base.Logger, ts *metrics.TelemetryServer, diff --git a/zetaclient/chains/solana/rpc/rpc.go b/zetaclient/chains/solana/rpc/rpc.go index 00c2add253..c62bcccb06 100644 --- a/zetaclient/chains/solana/rpc/rpc.go +++ b/zetaclient/chains/solana/rpc/rpc.go @@ -18,7 +18,7 @@ const ( // RPCAlertLatency is the default threshold for RPC latency to be considered unhealthy and trigger an alert. // The 'HEALTH_CHECK_SLOT_DISTANCE' is default to 150 slots, which is 150 * 0.4s = 60s - RPCAlertLatency = 60 + RPCAlertLatency = time.Duration(60) * time.Second ) // GetFirstSignatureForAddress searches the first signature for the given address. @@ -127,7 +127,7 @@ func GetSignaturesForAddressUntil( func CheckRPCStatus( ctx context.Context, client interfaces.SolanaRPCClient, - alertLatency uint64, + alertLatency time.Duration, logger zerolog.Logger, ) error { // query solana health (always return "ok" unless --trusted-validator is provided) @@ -149,21 +149,21 @@ func CheckRPCStatus( } // use default alert latency if not provided - if alertLatency == 0 { + if alertLatency <= 0 { alertLatency = RPCAlertLatency } // latest block should not be too old - elapsedSeconds := time.Since(blockTime.Time()).Seconds() - if elapsedSeconds > float64(alertLatency) { + elapsedTime := time.Since(blockTime.Time()) + if elapsedTime > alertLatency { return errors.Errorf( "Latest slot %d is %.0fs old, RPC stale or chain stuck (check explorer)?", slot, - elapsedSeconds, + elapsedTime.Seconds(), ) } logger.Info(). - Msgf("RPC Status [OK]: latest slot %d, timestamp %s (%.0fs ago)", slot, blockTime.String(), elapsedSeconds) + Msgf("RPC Status [OK]: latest slot %d, timestamp %s (%.0fs ago)", slot, blockTime.String(), elapsedTime.Seconds()) return nil } diff --git a/zetaclient/chains/solana/rpc/rpc_live_test.go b/zetaclient/chains/solana/rpc/rpc_live_test.go index b04352d944..35c1b6ae70 100644 --- a/zetaclient/chains/solana/rpc/rpc_live_test.go +++ b/zetaclient/chains/solana/rpc/rpc_live_test.go @@ -9,13 +9,18 @@ import ( "github.com/rs/zerolog/log" "github.com/stretchr/testify/require" "github.com/zeta-chain/zetacore/zetaclient/chains/solana/rpc" + "github.com/zeta-chain/zetacore/zetaclient/common" ) // Test_SolanaRPCLive is a phony test to run all live tests func Test_SolanaRPCLive(t *testing.T) { - // LiveTest_GetFirstSignatureForAddress(t) - // LiveTest_GetSignaturesForAddressUntil(t) - // LiveTest_CheckRPCStatus(t) + if !common.LiveTestEnabled() { + return + } + + LiveTest_GetFirstSignatureForAddress(t) + LiveTest_GetSignaturesForAddressUntil(t) + LiveTest_CheckRPCStatus(t) } func LiveTest_GetFirstSignatureForAddress(t *testing.T) { @@ -44,8 +49,8 @@ func LiveTest_GetSignaturesForAddressUntil(t *testing.T) { "2tUQtcrXxtNFtV9kZ4kQsmY7snnEoEEArmu9pUptr4UCy8UdbtjPD6UtfEtPJ2qk5CTzZTmLwsbmZdLymcwSUcHu", ) - // get all signatures for the address until the first signature (one by one) - sigs, err := rpc.GetSignaturesForAddressUntil(context.Background(), client, address, untilSig, 1) + // get all signatures for the address until the first signature + sigs, err := rpc.GetSignaturesForAddressUntil(context.Background(), client, address, untilSig, 100) require.NoError(t, err) // assert diff --git a/zetaclient/common/env.go b/zetaclient/common/env.go new file mode 100644 index 0000000000..ca48ef7615 --- /dev/null +++ b/zetaclient/common/env.go @@ -0,0 +1,24 @@ +package common + +import ( + "os" + "strings" +) + +const ( + // EnvEnableLiveTest is the environment variable to enable zetaclient live tests + EnvEnableLiveTest = "ENABLE_LIVE_TEST" + + // EnvBtcRPCMainnet is the environment variable to enable mainnet for bitcoin rpc + EnvBtcRPCMainnet = "BTC_RPC_MAINNET" + + // EnvBtcRPCTestnet is the environment variable to enable testnet for bitcoin rpc + EnvBtcRPCTestnet = "BTC_RPC_TESTNET" +) + +// LiveTestEnabled returns true if live tests are enabled +func LiveTestEnabled() bool { + value := os.Getenv(EnvEnableLiveTest) + + return strings.ToLower(value) == "true" || value == "1" +} diff --git a/zetaclient/config/types.go b/zetaclient/config/types.go index 37c8024b0f..55164efb33 100644 --- a/zetaclient/config/types.go +++ b/zetaclient/config/types.go @@ -36,7 +36,7 @@ type ClientConfiguration struct { type EVMConfig struct { Chain chains.Chain Endpoint string - RPCAlertLatency uint64 + RPCAlertLatency int64 } // BTCConfig is the config for Bitcoin chain @@ -46,13 +46,13 @@ type BTCConfig struct { RPCPassword string RPCHost string RPCParams string // "regtest", "mainnet", "testnet3" - RPCAlertLatency uint64 + RPCAlertLatency int64 } // SolanaConfig is the config for Solana chain type SolanaConfig struct { Endpoint string - RPCAlertLatency uint64 + RPCAlertLatency int64 } // ComplianceConfig is the config for compliance diff --git a/zetaclient/orchestrator/bootstrap.go b/zetaclient/orchestrator/bootstrap.go index e997eaeaa4..0f91a83dc6 100644 --- a/zetaclient/orchestrator/bootstrap.go +++ b/zetaclient/orchestrator/bootstrap.go @@ -2,6 +2,7 @@ package orchestrator import ( "context" + "time" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" @@ -341,7 +342,7 @@ func syncObserverMap( *params, client, tss, - cfg.RPCAlertLatency, + time.Duration(cfg.RPCAlertLatency), database, logger, ts, @@ -378,7 +379,7 @@ func syncObserverMap( *params, client, tss, - cfg.RPCAlertLatency, + time.Duration(cfg.RPCAlertLatency), database, logger, ts,