From f7656f5403482aaad72edcf8162569326cacecc5 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Mon, 8 Jul 2024 13:56:30 -0500 Subject: [PATCH 1/3] special handle bitcoin testnet gas price estimator --- changelog.md | 1 + .../chains/bitcoin/observer/observer.go | 48 ++++++++++++------ zetaclient/chains/bitcoin/rpc/rpc.go | 49 +++++++++++++++++++ .../chains/bitcoin/rpc/rpc_live_test.go | 34 +++++++++---- 4 files changed, 108 insertions(+), 24 deletions(-) diff --git a/changelog.md b/changelog.md index 7b679fb678..438b085002 100644 --- a/changelog.md +++ b/changelog.md @@ -89,6 +89,7 @@ * [2327](https://github.com/zeta-chain/node/pull/2327) - partially cherry picked the fix to Bitcoin outbound dust amount * [2362](https://github.com/zeta-chain/node/pull/2362) - set 1000 satoshis as minimum BTC amount that can be withdrawn from zEVM * [2382](https://github.com/zeta-chain/node/pull/2382) - add tx input and gas in rpc methods for synthetic eth txs +* [2396](https://github.com/zeta-chain/node/issues/2386) - special handle bitcoin testnet gas price estimator ### CI * [2388](https://github.com/zeta-chain/node/pull/2388) - added GitHub attestations of binaries produced in the release workflow. diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index a04188da80..21fa0fc21b 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -23,6 +23,7 @@ import ( observertypes "github.com/zeta-chain/zetacore/x/observer/types" "github.com/zeta-chain/zetacore/zetaclient/chains/base" "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin" + "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin/rpc" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" "github.com/zeta-chain/zetacore/zetaclient/context" "github.com/zeta-chain/zetacore/zetaclient/metrics" @@ -341,20 +342,9 @@ func (ob *Observer) WatchGasPrice() { // PostGasPrice posts gas price to zetacore // TODO(revamp): move to gas price file func (ob *Observer) PostGasPrice() error { - // hardcode gas price here since this RPC is not available on regtest - if chains.IsBitcoinRegnet(ob.Chain().ChainId) { - blockNumber, err := ob.btcClient.GetBlockCount() - if err != nil { - return err - } - - // #nosec G701 always in range - _, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), 1, "100", uint64(blockNumber)) - if err != nil { - ob.logger.GasPrice.Err(err).Msg("PostGasPrice:") - return err - } - return nil + // special handle regnet and testnet gas rate + if ob.Chain().NetworkType != chains.NetworkType_mainnet { + return ob.specialHandleFeeRate() } // EstimateSmartFee returns the fees per kilobyte (BTC/kb) targeting given block confirmation @@ -644,6 +634,36 @@ func (ob *Observer) LoadBroadcastedTxMap() error { return nil } +// specialHandleFeeRate handles the fee rate for regnet and testnet +func (ob *Observer) specialHandleFeeRate() error { + blockNumber, err := ob.btcClient.GetBlockCount() + if err != nil { + return errors.Wrapf(err, "specialHandleFeeRate: error GetBlockCount") + } + + feeRateEstimated := uint64(0) + switch ob.Chain().NetworkType { + case chains.NetworkType_privnet: + // hardcode gas price here since RPC 'EstimateSmartFee' is not available on regtest (privnet) + feeRateEstimated = 1 + case chains.NetworkType_testnet: + feeRateEstimated, err = rpc.GetRecentFeeRate(ob.btcClient, ob.netParams) + if err != nil { + return errors.Wrapf(err, "specialHandleFeeRate: error GetRecentFeeRate") + } + default: + return fmt.Errorf("specialHandleFeeRate: unsupported bitcoin network type %d", ob.Chain().NetworkType) + } + + // #nosec G701 always in range + _, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), feeRateEstimated, "100", uint64(blockNumber)) + if err != nil { + return errors.Wrapf(err, "specialHandleFeeRate: error PostGasPrice") + } + + return nil +} + // isTssTransaction checks if a given transaction was sent by TSS itself. // An unconfirmed transaction is safe to spend only if it was sent by TSS and verified by ourselves. func (ob *Observer) isTssTransaction(txid string) bool { diff --git a/zetaclient/chains/bitcoin/rpc/rpc.go b/zetaclient/chains/bitcoin/rpc/rpc.go index 32144d77f2..1fcdea114d 100644 --- a/zetaclient/chains/bitcoin/rpc/rpc.go +++ b/zetaclient/chains/bitcoin/rpc/rpc.go @@ -4,14 +4,24 @@ import ( "fmt" "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/rpcclient" "github.com/pkg/errors" + "github.com/zeta-chain/zetacore/zetaclient/chains/bitcoin" "github.com/zeta-chain/zetacore/zetaclient/chains/interfaces" "github.com/zeta-chain/zetacore/zetaclient/config" ) +const ( + // feeRateCountBackBlocks is the default number of blocks to look back for fee rate estimation + feeRateCountBackBlocks = 2 + + // defaultTestnetFeeRate is the default fee rate for testnet, 10 sat/byte + defaultTestnetFeeRate = 10 +) + // NewRPCClient creates a new RPC client by the given config. func NewRPCClient(btcConfig config.BTCConfig) (*rpcclient.Client, error) { connCfg := &rpcclient.ConnConfig{ @@ -107,3 +117,42 @@ func GetRawTxResult( // res.Confirmations < 0 (meaning not included) return btcjson.TxRawResult{}, fmt.Errorf("GetRawTxResult: tx %s not included yet", hash) } + +// GetRecentFeeRate gets the highest fee rate from recent blocks +func GetRecentFeeRate(rpcClient interfaces.BTCRPCClient, netParams *chaincfg.Params) (uint64, error) { + blockNumber, err := rpcClient.GetBlockCount() + if err != nil { + return 0, err + } + + // get the highest fee rate among recent 'countBack' blocks to avoid underestimation + highestRate := int64(0) + for i := int64(0); i < feeRateCountBackBlocks; i++ { + // get the block + hash, err := rpcClient.GetBlockHash(blockNumber - i) + if err != nil { + return 0, err + } + block, err := rpcClient.GetBlockVerboseTx(hash) + if err != nil { + return 0, err + } + + // computes the average fee rate of the block and take the higher rate + avgFeeRate, err := bitcoin.CalcBlockAvgFeeRate(block, netParams) + if err != nil { + return 0, err + } + if avgFeeRate > highestRate { + highestRate = avgFeeRate + } + } + + // use 10 sat/byte as default estimation when fee rate is 0 + if highestRate == 0 { + highestRate = defaultTestnetFeeRate + } + + // #nosec G701 always in range + return uint64(highestRate), nil +} diff --git a/zetaclient/chains/bitcoin/rpc/rpc_live_test.go b/zetaclient/chains/bitcoin/rpc/rpc_live_test.go index 97a373f94d..7f319871a6 100644 --- a/zetaclient/chains/bitcoin/rpc/rpc_live_test.go +++ b/zetaclient/chains/bitcoin/rpc/rpc_live_test.go @@ -59,7 +59,7 @@ func (suite *BitcoinObserverTestSuite) SetupTest() { base.DefaultLogger(), nil) suite.Require().NoError(err) suite.Require().NotNil(ob) - suite.rpcClient, err = getRPCClient(18332) + suite.rpcClient, err = createRPCClient(18332) suite.Require().NoError(err) skBytes, err := hex.DecodeString(skHex) suite.Require().NoError(err) @@ -91,13 +91,14 @@ func (suite *BitcoinObserverTestSuite) SetupTest() { func (suite *BitcoinObserverTestSuite) TearDownSuite() { } -func getRPCClient(chainID int64) (*rpcclient.Client, error) { +// createRPCClient creates a new Bitcoin RPC client for given chainID +func createRPCClient(chainID int64) (*rpcclient.Client, error) { var connCfg *rpcclient.ConnConfig rpcMainnet := os.Getenv("BTC_RPC_MAINNET") rpcTestnet := os.Getenv("BTC_RPC_TESTNET") // mainnet - if chainID == 8332 { + if chainID == chains.BitcoinMainnet.ChainId { connCfg = &rpcclient.ConnConfig{ Host: rpcMainnet, // mainnet endpoint goes here User: "user", @@ -108,7 +109,7 @@ func getRPCClient(chainID int64) (*rpcclient.Client, error) { } } // testnet3 - if chainID == 18332 { + if chainID == chains.BitcoinTestnet.ChainId { connCfg = &rpcclient.ConnConfig{ Host: rpcTestnet, // testnet endpoint goes here User: "user", @@ -218,6 +219,7 @@ func TestBitcoinObserverLive(t *testing.T) { // LiveTestBitcoinFeeRate(t) // LiveTestAvgFeeRateMainnetMempoolSpace(t) // LiveTestAvgFeeRateTestnetMempoolSpace(t) + // LiveTestGetRecentFeeRate(t) // LiveTestGetSenderByVin(t) } @@ -243,7 +245,7 @@ func LiveTestNewRPCClient(t *testing.T) { // LiveTestGetBlockHeightByHash queries Bitcoin block height by hash func LiveTestGetBlockHeightByHash(t *testing.T) { // setup Bitcoin client - client, err := getRPCClient(8332) + client, err := createRPCClient(chains.BitcoinMainnet.ChainId) require.NoError(t, err) // the block hashes to test @@ -265,7 +267,7 @@ func LiveTestGetBlockHeightByHash(t *testing.T) { // and compares Conservative and Economical fee rates for different block targets (1 and 2) func LiveTestBitcoinFeeRate(t *testing.T) { // setup Bitcoin client - client, err := getRPCClient(8332) + client, err := createRPCClient(chains.BitcoinMainnet.ChainId) require.NoError(t, err) bn, err := client.GetBlockCount() if err != nil { @@ -390,7 +392,7 @@ func compareAvgFeeRate(t *testing.T, client *rpcclient.Client, startBlock int, e // LiveTestAvgFeeRateMainnetMempoolSpace compares calculated fee rate with mempool.space fee rate for mainnet func LiveTestAvgFeeRateMainnetMempoolSpace(t *testing.T) { // setup Bitcoin client - client, err := getRPCClient(8332) + client, err := createRPCClient(chains.BitcoinMainnet.ChainId) require.NoError(t, err) // test against mempool.space API for 10000 blocks @@ -404,7 +406,7 @@ func LiveTestAvgFeeRateMainnetMempoolSpace(t *testing.T) { // LiveTestAvgFeeRateTestnetMempoolSpace compares calculated fee rate with mempool.space fee rate for testnet func LiveTestAvgFeeRateTestnetMempoolSpace(t *testing.T) { // setup Bitcoin client - client, err := getRPCClient(18332) + client, err := createRPCClient(chains.BitcoinTestnet.ChainId) require.NoError(t, err) // test against mempool.space API for 10000 blocks @@ -415,11 +417,23 @@ func LiveTestAvgFeeRateTestnetMempoolSpace(t *testing.T) { compareAvgFeeRate(t, client, startBlock, endBlock, true) } +// LiveTestGetRecentFeeRate gets the highest fee rate from recent blocks +func LiveTestGetRecentFeeRate(t *testing.T) { + // setup Bitcoin testnet client + client, err := createRPCClient(chains.BitcoinTestnet.ChainId) + require.NoError(t, err) + + // get fee rate from recent blocks + feeRate, err := rpc.GetRecentFeeRate(client, &chaincfg.TestNet3Params) + require.NoError(t, err) + require.Greater(t, feeRate, uint64(0)) +} + // LiveTestGetSenderByVin gets sender address for each vin and compares with mempool.space sender address func LiveTestGetSenderByVin(t *testing.T) { // setup Bitcoin client - chainID := int64(8332) - client, err := getRPCClient(chainID) + chainID := chains.BitcoinMainnet.ChainId + client, err := createRPCClient(chainID) require.NoError(t, err) // net params From 3dc39b63a44ac0eefa8b765b4120fc946d5f32b5 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 10 Jul 2024 11:32:19 -0500 Subject: [PATCH 2/3] let specialHandleFeeRate return the fee rate number and calls PostGasPrice in single spot --- .../chains/bitcoin/observer/observer.go | 66 +++++++++---------- zetaclient/chains/bitcoin/rpc/rpc.go | 3 +- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 21fa0fc21b..72b6a517b0 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -342,33 +342,42 @@ func (ob *Observer) WatchGasPrice() { // PostGasPrice posts gas price to zetacore // TODO(revamp): move to gas price file func (ob *Observer) PostGasPrice() error { + var err error + feeRateEstimated := uint64(0) + // special handle regnet and testnet gas rate if ob.Chain().NetworkType != chains.NetworkType_mainnet { - return ob.specialHandleFeeRate() - } - - // EstimateSmartFee returns the fees per kilobyte (BTC/kb) targeting given block confirmation - feeResult, err := ob.btcClient.EstimateSmartFee(1, &btcjson.EstimateModeEconomical) - if err != nil { - return err - } - if feeResult.Errors != nil || feeResult.FeeRate == nil { - return fmt.Errorf("error getting gas price: %s", feeResult.Errors) - } - if *feeResult.FeeRate > math.MaxInt64 { - return fmt.Errorf("gas price is too large: %f", *feeResult.FeeRate) + feeRateEstimated, err = ob.specialHandleFeeRate() + if err != nil { + ob.logger.GasPrice.Err(err).Msg("error specialHandleFeeRate") + return err + } + } else { + // EstimateSmartFee returns the fees per kilobyte (BTC/kb) targeting given block confirmation + feeResult, err := ob.btcClient.EstimateSmartFee(1, &btcjson.EstimateModeEconomical) + if err != nil { + ob.logger.GasPrice.Err(err).Msg("error EstimateSmartFee") + return err + } + if feeResult.Errors != nil || feeResult.FeeRate == nil { + return fmt.Errorf("error getting gas price: %s", feeResult.Errors) + } + if *feeResult.FeeRate > math.MaxInt64 { + return fmt.Errorf("gas price is too large: %f", *feeResult.FeeRate) + } + feeRateEstimated = bitcoin.FeeRateToSatPerByte(*feeResult.FeeRate).Uint64() } - feeRatePerByte := bitcoin.FeeRateToSatPerByte(*feeResult.FeeRate) + // query the current block number blockNumber, err := ob.btcClient.GetBlockCount() if err != nil { return err } // #nosec G701 always positive - _, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), feeRatePerByte.Uint64(), "100", uint64(blockNumber)) + _, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), feeRateEstimated, "100", uint64(blockNumber)) if err != nil { - ob.logger.GasPrice.Err(err).Msg("PostGasPrice:") + ob.logger.GasPrice.Err(err).Msg("err PostGasPrice") return err } @@ -635,33 +644,20 @@ func (ob *Observer) LoadBroadcastedTxMap() error { } // specialHandleFeeRate handles the fee rate for regnet and testnet -func (ob *Observer) specialHandleFeeRate() error { - blockNumber, err := ob.btcClient.GetBlockCount() - if err != nil { - return errors.Wrapf(err, "specialHandleFeeRate: error GetBlockCount") - } - - feeRateEstimated := uint64(0) +func (ob *Observer) specialHandleFeeRate() (uint64, error) { switch ob.Chain().NetworkType { case chains.NetworkType_privnet: // hardcode gas price here since RPC 'EstimateSmartFee' is not available on regtest (privnet) - feeRateEstimated = 1 + return 1, nil case chains.NetworkType_testnet: - feeRateEstimated, err = rpc.GetRecentFeeRate(ob.btcClient, ob.netParams) + feeRateEstimated, err := rpc.GetRecentFeeRate(ob.btcClient, ob.netParams) if err != nil { - return errors.Wrapf(err, "specialHandleFeeRate: error GetRecentFeeRate") + return 0, errors.Wrapf(err, "error GetRecentFeeRate") } + return feeRateEstimated, nil default: - return fmt.Errorf("specialHandleFeeRate: unsupported bitcoin network type %d", ob.Chain().NetworkType) + return 0, fmt.Errorf(" unsupported bitcoin network type %d", ob.Chain().NetworkType) } - - // #nosec G701 always in range - _, err = ob.ZetacoreClient().PostGasPrice(ob.Chain(), feeRateEstimated, "100", uint64(blockNumber)) - if err != nil { - return errors.Wrapf(err, "specialHandleFeeRate: error PostGasPrice") - } - - return nil } // isTssTransaction checks if a given transaction was sent by TSS itself. diff --git a/zetaclient/chains/bitcoin/rpc/rpc.go b/zetaclient/chains/bitcoin/rpc/rpc.go index 1fcdea114d..6b468c87ed 100644 --- a/zetaclient/chains/bitcoin/rpc/rpc.go +++ b/zetaclient/chains/bitcoin/rpc/rpc.go @@ -119,6 +119,7 @@ func GetRawTxResult( } // GetRecentFeeRate gets the highest fee rate from recent blocks +// Note: this method is only used for testnet func GetRecentFeeRate(rpcClient interfaces.BTCRPCClient, netParams *chaincfg.Params) (uint64, error) { blockNumber, err := rpcClient.GetBlockCount() if err != nil { @@ -148,7 +149,7 @@ func GetRecentFeeRate(rpcClient interfaces.BTCRPCClient, netParams *chaincfg.Par } } - // use 10 sat/byte as default estimation when fee rate is 0 + // use 10 sat/byte as default estimation if recent fee rate drops to 0 if highestRate == 0 { highestRate = defaultTestnetFeeRate } From 570fe7efb98b8897599e6d8d83844b1be01d7a94 Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Wed, 10 Jul 2024 12:52:09 -0500 Subject: [PATCH 3/3] added some explanations for specialHandleFeeRate --- zetaclient/chains/bitcoin/observer/observer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zetaclient/chains/bitcoin/observer/observer.go b/zetaclient/chains/bitcoin/observer/observer.go index 72b6a517b0..b2781380b1 100644 --- a/zetaclient/chains/bitcoin/observer/observer.go +++ b/zetaclient/chains/bitcoin/observer/observer.go @@ -346,6 +346,8 @@ func (ob *Observer) PostGasPrice() error { feeRateEstimated := uint64(0) // special handle regnet and testnet gas rate + // regnet: RPC 'EstimateSmartFee' is not available + // testnet: RPC 'EstimateSmartFee' returns unreasonable high gas rate if ob.Chain().NetworkType != chains.NetworkType_mainnet { feeRateEstimated, err = ob.specialHandleFeeRate() if err != nil { @@ -647,7 +649,7 @@ func (ob *Observer) LoadBroadcastedTxMap() error { func (ob *Observer) specialHandleFeeRate() (uint64, error) { switch ob.Chain().NetworkType { case chains.NetworkType_privnet: - // hardcode gas price here since RPC 'EstimateSmartFee' is not available on regtest (privnet) + // hardcode gas price for regnet return 1, nil case chains.NetworkType_testnet: feeRateEstimated, err := rpc.GetRecentFeeRate(ob.btcClient, ob.netParams)