diff --git a/common/default_chains_mainnet.go b/common/default_chains_mainnet.go index 3b80f2a89e..62b455e571 100644 --- a/common/default_chains_mainnet.go +++ b/common/default_chains_mainnet.go @@ -36,7 +36,7 @@ func BtcChainID() int64 { } func BtcDustOffset() int64 { - return 0 + return 2000 } func PolygonChain() Chain { diff --git a/zetaclient/bitcoin_client.go b/zetaclient/bitcoin_client.go index 4fc07e89bd..fd360a3453 100644 --- a/zetaclient/bitcoin_client.go +++ b/zetaclient/bitcoin_client.go @@ -815,12 +815,17 @@ func (ob *BitcoinChainClient) findNonceMarkUTXO(nonce uint64, txid string) (int, // // Parameters: // - amount: The desired minimum total value of the selected UTXOs. -// - utxoCap: The maximum number of UTXOs to be selected. +// - utxos2Spend: The maximum number of UTXOs to spend. // - nonce: The nonce of the outbound transaction. +// - consolidateRank: The rank below which UTXOs will be consolidated. // - test: true for unit test only. // -// Returns: a sublist (includes previous nonce-mark) of UTXOs or an error if the qulifying sublist cannot be found. -func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce uint64, test bool) ([]btcjson.ListUnspentResult, float64, error) { +// Returns: +// - a sublist (includes previous nonce-mark) of UTXOs or an error if the qualifying sublist cannot be found. +// - the total value of the selected UTXOs. +// - the number of consolidated UTXOs. +// - the total value of the consolidated UTXOs. +func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxosToSpend uint16, nonce uint64, consolidateRank uint16, test bool) ([]btcjson.ListUnspentResult, float64, uint16, float64, error) { idx := -1 if nonce == 0 { // for nonce = 0; make exception; no need to include nonce-mark utxo @@ -830,24 +835,24 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u // for nonce > 0; we proceed only when we see the nonce-mark utxo preTxid, err := ob.getOutTxidByNonce(nonce-1, test) if err != nil { - return nil, 0, err + return nil, 0, 0, 0, err } ob.Mu.Lock() defer ob.Mu.Unlock() idx, err = ob.findNonceMarkUTXO(nonce-1, preTxid) if err != nil { - return nil, 0, err + return nil, 0, 0, 0, err } } - // select utxos + // select smallest possible UTXOs to make payment total := 0.0 left, right := 0, 0 for total < amount && right < len(ob.utxos) { - if utxoCap > 0 { // expand sublist + if utxosToSpend > 0 { // expand sublist total += ob.utxos[right].Amount right++ - utxoCap-- + utxosToSpend-- } else { // pop the smallest utxo and append the current one total -= ob.utxos[left].Amount total += ob.utxos[right].Amount @@ -858,8 +863,8 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u results := make([]btcjson.ListUnspentResult, right-left) copy(results, ob.utxos[left:right]) - // Note: always put nonce-mark as 1st input - if idx >= 0 { + // include nonce-mark as the 1st input + if idx >= 0 { // for nonce > 0 if idx < left || idx >= right { total += ob.utxos[idx].Amount results = append([]btcjson.ListUnspentResult{ob.utxos[idx]}, results...) @@ -870,9 +875,26 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u } } if total < amount { - return nil, 0, fmt.Errorf("SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", total, amount) + return nil, 0, 0, 0, fmt.Errorf("SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", total, amount) + } + + // consolidate biggest possible UTXOs to maximize consolidated value + // consolidation happens only when there are more than (or equal to) consolidateRank (10) UTXOs + utxoRank, consolidatedUtxo, consolidatedValue := uint16(0), uint16(0), 0.0 + for i := len(ob.utxos) - 1; i >= 0 && utxosToSpend > 0; i-- { // iterate over UTXOs big-to-small + if i != idx && (i < left || i >= right) { // exclude nonce-mark and already selected UTXOs + utxoRank++ + if utxoRank >= consolidateRank { // consolication starts from the 10-ranked UTXO based on value + utxosToSpend-- + consolidatedUtxo++ + total += ob.utxos[i].Amount + consolidatedValue += ob.utxos[i].Amount + results = append(results, ob.utxos[i]) + } + } } - return results, total, nil + + return results, total, consolidatedUtxo, consolidatedValue, nil } // Save successfully broadcasted transaction diff --git a/zetaclient/btc_signer.go b/zetaclient/btc_signer.go index f426fee681..afe93c7d4e 100644 --- a/zetaclient/btc_signer.go +++ b/zetaclient/btc_signer.go @@ -23,8 +23,9 @@ import ( const ( maxNoOfInputsPerTx = 20 - outTxBytesMin = 400 // 500B is a conservative estimate for a 2-input, 3-output SegWit tx - outTxBytesMax = 4_000 // 4KB is a conservative estimate for a 21-input, 3-output SegWit tx + consolidationRank = 10 // the rank below (or equal to) which we consolidate UTXOs + outTxBytesMin = 400 // 500B is an estimated size for a 2-input, 3-output SegWit tx + outTxBytesMax = 3250 // 3250B is an estimated size for a 21-input, 3-output SegWit tx outTxBytesCap = 10_000 // in case of accident // for ZRC20 configuration @@ -68,9 +69,7 @@ func NewBTCSigner(cfg config.BTCConfig, tssSigner TSSSigner, logger zerolog.Logg // SignWithdrawTx receives utxos sorted by value, amount in BTC, feeRate in BTC per Kb func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, amount float64, gasPrice *big.Int, sizeLimit uint64, btcClient *BitcoinChainClient, height uint64, nonce uint64, chain *common.Chain) (*wire.MsgTx, error) { - estimateFee := float64(gasPrice.Uint64()) * outTxBytesMax / 1e8 - nonceMark := common.NonceMarkAmount(nonce) // refresh unspent UTXOs and continue with keysign regardless of error @@ -80,7 +79,7 @@ func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, am } // select N UTXOs to cover the total expense - prevOuts, total, err := btcClient.SelectUTXOs(amount+estimateFee+float64(nonceMark)*1e-8, maxNoOfInputsPerTx, nonce, false) + prevOuts, total, consolidatedUtxo, consolidatedValue, err := btcClient.SelectUTXOs(amount+estimateFee+float64(nonceMark)*1e-8, maxNoOfInputsPerTx, nonce, consolidationRank, false) if err != nil { return nil, err } @@ -120,7 +119,8 @@ func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, am // fee calculation // #nosec G701 always in range (checked above) fees := new(big.Int).Mul(big.NewInt(int64(txSize)), gasPrice) - signer.logger.Info().Msgf("bitcoin outTx nonce %d gasPrice %s size %d fees %s", nonce, gasPrice.String(), txSize, fees.String()) + signer.logger.Info().Msgf("bitcoin outTx nonce %d gasPrice %s size %d fees %s consolidated %d utxos of value %v", + nonce, gasPrice.String(), txSize, fees.String(), consolidatedUtxo, consolidatedValue) // calculate remaining btc to TSS self tssAddrWPKH := signer.tssSigner.BTCAddressWitnessPubkeyHash() diff --git a/zetaclient/btc_signer_test.go b/zetaclient/btc_signer_test.go index 5f3eb9a250..38b648ebec 100644 --- a/zetaclient/btc_signer_test.go +++ b/zetaclient/btc_signer_test.go @@ -3,6 +3,7 @@ package zetaclient import ( "encoding/hex" "fmt" + "math" "sort" "sync" "testing" @@ -232,25 +233,27 @@ func mineTxNSetNonceMark(ob *BitcoinChainClient, nonce uint64, txid string, preM ob.includedTxResults[outTxID] = btcjson.GetTransactionResult{TxID: txid} // Set nonce mark - if preMarkIndex >= 0 { - tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress() - nonceMark := btcjson.ListUnspentResult{TxID: txid, Address: tssAddress, Amount: float64(common.NonceMarkAmount(nonce)) * 1e-8} + tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress() + nonceMark := btcjson.ListUnspentResult{TxID: txid, Address: tssAddress, Amount: float64(common.NonceMarkAmount(nonce)) * 1e-8} + if preMarkIndex >= 0 { // replace nonce-mark utxo ob.utxos[preMarkIndex] = nonceMark - sort.SliceStable(ob.utxos, func(i, j int) bool { - return ob.utxos[i].Amount < ob.utxos[j].Amount - }) + + } else { // add nonce-mark utxo directly + ob.utxos = append(ob.utxos, nonceMark) } + sort.SliceStable(ob.utxos, func(i, j int) bool { + return ob.utxos[i].Amount < ob.utxos[j].Amount + }) } func TestSelectUTXOs(t *testing.T) { ob := createTestClient(t) - tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress() dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4" // Case1: nonce = 0, bootstrap // input: utxoCap = 5, amount = 0.01, nonce = 0 // output: [0.01], 0.01 - result, amount, err := ob.SelectUTXOs(0.01, 5, 0, true) + result, amount, _, _, err := ob.SelectUTXOs(0.01, 5, 0, math.MaxUint16, true) require.Nil(t, err) require.Equal(t, 0.01, amount) require.Equal(t, ob.utxos[0:1], result) @@ -258,89 +261,184 @@ func TestSelectUTXOs(t *testing.T) { // Case2: nonce = 1, must FAIL and wait for previous transaction to be mined // input: utxoCap = 5, amount = 0.5, nonce = 1 // output: error - result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true) + result, amount, _, _, err = ob.SelectUTXOs(0.5, 5, 1, math.MaxUint16, true) require.NotNil(t, err) require.Nil(t, result) require.Zero(t, amount) require.Equal(t, "getOutTxidByNonce: cannot find outTx txid for nonce 0", err.Error()) - mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction for nonce 0 - - // Case3: nonce = 1, must FAIL without nonce mark utxo - // input: utxoCap = 5, amount = 0.5, nonce = 1 - // output: error - result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true) - require.NotNil(t, err) - require.Nil(t, result) - require.Zero(t, amount) - require.Equal(t, "findNonceMarkUTXO: cannot find nonce-mark utxo with nonce 0", err.Error()) + mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 - // add nonce-mark utxo for nonce 0 - nonceMark0 := btcjson.ListUnspentResult{TxID: dummyTxID, Address: tssAddress, Amount: float64(common.NonceMarkAmount(0)) * 1e-8} - ob.utxos = append([]btcjson.ListUnspentResult{nonceMark0}, ob.utxos...) - - // Case4: nonce = 1, should pass now + // Case3: nonce = 1, should pass now // input: utxoCap = 5, amount = 0.5, nonce = 1 // output: [0.00002, 0.01, 0.12, 0.18, 0.24], 0.55002 - result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true) + result, amount, _, _, err = ob.SelectUTXOs(0.5, 5, 1, math.MaxUint16, true) require.Nil(t, err) require.Equal(t, 0.55002, amount) require.Equal(t, ob.utxos[0:5], result) mineTxNSetNonceMark(ob, 1, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 1 - // Case5: + // Case4: // input: utxoCap = 5, amount = 1.0, nonce = 2 // output: [0.00002001, 0.01, 0.12, 0.18, 0.24, 0.5], 1.05002001 - result, amount, err = ob.SelectUTXOs(1.0, 5, 2, true) + result, amount, _, _, err = ob.SelectUTXOs(1.0, 5, 2, math.MaxUint16, true) require.Nil(t, err) assert.InEpsilon(t, 1.05002001, amount, 1e-8) require.Equal(t, ob.utxos[0:6], result) mineTxNSetNonceMark(ob, 2, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 2 - // Case6: should include nonce-mark utxo on the LEFT + // Case5: should include nonce-mark utxo on the LEFT // input: utxoCap = 5, amount = 8.05, nonce = 3 // output: [0.00002002, 0.24, 0.5, 1.26, 2.97, 3.28], 8.25002002 - result, amount, err = ob.SelectUTXOs(8.05, 5, 3, true) + result, amount, _, _, err = ob.SelectUTXOs(8.05, 5, 3, math.MaxUint16, true) require.Nil(t, err) assert.InEpsilon(t, 8.25002002, amount, 1e-8) expected := append([]btcjson.ListUnspentResult{ob.utxos[0]}, ob.utxos[4:9]...) require.Equal(t, expected, result) mineTxNSetNonceMark(ob, 24105431, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 24105431 - // Case7: should include nonce-mark utxo on the RIGHT + // Case6: should include nonce-mark utxo on the RIGHT // input: utxoCap = 5, amount = 0.503, nonce = 24105432 // output: [0.24107432, 0.01, 0.12, 0.18, 0.24], 0.55002002 - result, amount, err = ob.SelectUTXOs(0.503, 5, 24105432, true) + result, amount, _, _, err = ob.SelectUTXOs(0.503, 5, 24105432, math.MaxUint16, true) require.Nil(t, err) assert.InEpsilon(t, 0.79107431, amount, 1e-8) expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:4]...) require.Equal(t, expected, result) mineTxNSetNonceMark(ob, 24105432, dummyTxID, 4) // mine a transaction and set nonce-mark utxo for nonce 24105432 - // Case8: should include nonce-mark utxo in the MIDDLE + // Case7: should include nonce-mark utxo in the MIDDLE // input: utxoCap = 5, amount = 1.0, nonce = 24105433 // output: [0.24107432, 0.12, 0.18, 0.24, 0.5], 1.28107432 - result, amount, err = ob.SelectUTXOs(1.0, 5, 24105433, true) + result, amount, _, _, err = ob.SelectUTXOs(1.0, 5, 24105433, math.MaxUint16, true) require.Nil(t, err) assert.InEpsilon(t, 1.28107432, amount, 1e-8) expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[1:4]...) expected = append(expected, ob.utxos[5]) require.Equal(t, expected, result) - // Case9: should work with maximum amount + // Case8: should work with maximum amount // input: utxoCap = 5, amount = 16.03 // output: [0.24107432, 1.26, 2.97, 3.28, 5.16, 8.72], 21.63107432 - result, amount, err = ob.SelectUTXOs(16.03, 5, 24105433, true) + result, amount, _, _, err = ob.SelectUTXOs(16.03, 5, 24105433, math.MaxUint16, true) require.Nil(t, err) assert.InEpsilon(t, 21.63107432, amount, 1e-8) expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[6:11]...) require.Equal(t, expected, result) - // Case10: must FAIL due to insufficient funds + // Case9: must FAIL due to insufficient funds // input: utxoCap = 5, amount = 21.64 // output: error - result, amount, err = ob.SelectUTXOs(21.64, 5, 24105433, true) + result, amount, _, _, err = ob.SelectUTXOs(21.64, 5, 24105433, math.MaxUint16, true) require.NotNil(t, err) require.Nil(t, result) require.Zero(t, amount) require.Equal(t, "SelectUTXOs: not enough btc in reserve - available : 21.63107432 , tx amount : 21.64", err.Error()) } + +func TestUTXOConsolidation(t *testing.T) { + dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4" + + t.Run("should not consolidate", func(t *testing.T) { + ob := createTestClient(t) + mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 + + // input: utxoCap = 10, amount = 0.01, nonce = 1, rank = 10 + // output: [0.00002, 0.01], 0.01002 + result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 10, 1, 10, true) + require.Nil(t, err) + require.Equal(t, 0.01002, amount) + require.Equal(t, ob.utxos[0:2], result) + require.Equal(t, uint16(0), clsdtUtxo) + require.Equal(t, 0.0, clsdtValue) + }) + + t.Run("should consolidate 1 utxo", func(t *testing.T) { + ob := createTestClient(t) + mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 + + // input: utxoCap = 9, amount = 0.01, nonce = 1, rank = 9 + // output: [0.00002, 0.01, 0.12], 0.13002 + result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 9, 1, 9, true) + require.Nil(t, err) + require.Equal(t, 0.13002, amount) + require.Equal(t, ob.utxos[0:3], result) + require.Equal(t, uint16(1), clsdtUtxo) + require.Equal(t, 0.12, clsdtValue) + }) + + t.Run("should consolidate 3 utxos", func(t *testing.T) { + ob := createTestClient(t) + mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 + + // input: utxoCap = 5, amount = 0.01, nonce = 0, rank = 5 + // output: [0.00002, 0.014, 1.26, 0.5, 0.2], 2.01002 + result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 5, 1, 5, true) + require.Nil(t, err) + require.Equal(t, 2.01002, amount) + expected := make([]btcjson.ListUnspentResult, 2) + copy(expected, ob.utxos[0:2]) + for i := 6; i >= 4; i-- { // append consolidated utxos in descending order + expected = append(expected, ob.utxos[i]) + } + require.Equal(t, expected, result) + require.Equal(t, uint16(3), clsdtUtxo) + require.Equal(t, 2.0, clsdtValue) + }) + + t.Run("should consolidate all utxos using rank 1", func(t *testing.T) { + ob := createTestClient(t) + mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0 + + // input: utxoCap = 12, amount = 0.01, nonce = 0, rank = 1 + // output: [0.00002, 0.01, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18, 0.12], 22.44002 + result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 12, 1, 1, true) + require.Nil(t, err) + require.Equal(t, 22.44002, amount) + expected := make([]btcjson.ListUnspentResult, 2) + copy(expected, ob.utxos[0:2]) + for i := 10; i >= 2; i-- { // append consolidated utxos in descending order + expected = append(expected, ob.utxos[i]) + } + require.Equal(t, expected, result) + require.Equal(t, uint16(9), clsdtUtxo) + require.Equal(t, 22.43, clsdtValue) + }) + + t.Run("should consolidate 3 utxos sparse", func(t *testing.T) { + ob := createTestClient(t) + mineTxNSetNonceMark(ob, 24105431, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 24105431 + + // input: utxoCap = 5, amount = 0.13, nonce = 24105432, rank = 5 + // output: [0.24107431, 0.01, 0.12, 1.26, 0.5, 0.24], 2.37107431 + result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.13, 5, 24105432, 5, true) + require.Nil(t, err) + assert.InEpsilon(t, 2.37107431, amount, 1e-8) + expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...) + expected = append(expected, ob.utxos[6]) + expected = append(expected, ob.utxos[5]) + expected = append(expected, ob.utxos[3]) + require.Equal(t, expected, result) + require.Equal(t, uint16(3), clsdtUtxo) + require.Equal(t, 2.0, clsdtValue) + }) + + t.Run("should consolidate all utxos sparse", func(t *testing.T) { + ob := createTestClient(t) + mineTxNSetNonceMark(ob, 24105431, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 24105431 + + // input: utxoCap = 12, amount = 0.13, nonce = 24105432, rank = 1 + // output: [0.24107431, 0.01, 0.12, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18], 22.68107431 + result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.13, 12, 24105432, 1, true) + require.Nil(t, err) + assert.InEpsilon(t, 22.68107431, amount, 1e-8) + expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...) + for i := 10; i >= 5; i-- { // append consolidated utxos in descending order + expected = append(expected, ob.utxos[i]) + } + expected = append(expected, ob.utxos[3]) + expected = append(expected, ob.utxos[2]) + require.Equal(t, expected, result) + require.Equal(t, uint16(8), clsdtUtxo) + require.Equal(t, 22.31, clsdtValue) + }) +}