Skip to content

Commit

Permalink
feat: Bitcoin utxo consolidation (#1326)
Browse files Browse the repository at this point in the history 
* initiated utxo consolidation

* use real world max outTx size

* set consolidation rank to 10 to create a buffer (10) for small UTXOs received from Bitcoin omni-contract call

* revert DustOffset to 2000 for mainnet

* improved variable names and comments

---------

Co-authored-by: charliec <[email protected]>
  • Loading branch information
@ws4charlie
ws4charlie and ws4charlie authored Oct 23, 2023
1 parent d41cf7f commit a4f2978
Show file tree
Hide file tree
Showing 4 changed files with 176 additions and 56 deletions.
2 changes: 1 addition & 1 deletion common/default_chains_mainnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func BtcChainID() int64 {
}

func BtcDustOffset() int64 {
return 0
return 2000
}

func PolygonChain() Chain {
Expand Down
46 changes: 34 additions & 12 deletions zetaclient/bitcoin_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -815,12 +815,17 @@ func (ob *BitcoinChainClient) findNonceMarkUTXO(nonce uint64, txid string) (int,
//
// Parameters:
// - amount: The desired minimum total value of the selected UTXOs.
// - utxoCap: The maximum number of UTXOs to be selected.
// - utxos2Spend: The maximum number of UTXOs to spend.
// - nonce: The nonce of the outbound transaction.
// - consolidateRank: The rank below which UTXOs will be consolidated.
// - test: true for unit test only.
//
// Returns: a sublist (includes previous nonce-mark) of UTXOs or an error if the qulifying sublist cannot be found.
func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce uint64, test bool) ([]btcjson.ListUnspentResult, float64, error) {
// Returns:
// - a sublist (includes previous nonce-mark) of UTXOs or an error if the qualifying sublist cannot be found.
// - the total value of the selected UTXOs.
// - the number of consolidated UTXOs.
// - the total value of the consolidated UTXOs.
func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxosToSpend uint16, nonce uint64, consolidateRank uint16, test bool) ([]btcjson.ListUnspentResult, float64, uint16, float64, error) {
idx := -1
if nonce == 0 {
// for nonce = 0; make exception; no need to include nonce-mark utxo
Expand All @@ -830,24 +835,24 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u
// for nonce > 0; we proceed only when we see the nonce-mark utxo
preTxid, err := ob.getOutTxidByNonce(nonce-1, test)
if err != nil {
return nil, 0, err
return nil, 0, 0, 0, err
}
ob.Mu.Lock()
defer ob.Mu.Unlock()
idx, err = ob.findNonceMarkUTXO(nonce-1, preTxid)
if err != nil {
return nil, 0, err
return nil, 0, 0, 0, err
}
}

// select utxos
// select smallest possible UTXOs to make payment
total := 0.0
left, right := 0, 0
for total < amount && right < len(ob.utxos) {
if utxoCap > 0 { // expand sublist
if utxosToSpend > 0 { // expand sublist
total += ob.utxos[right].Amount
right++
utxoCap--
utxosToSpend--
} else { // pop the smallest utxo and append the current one
total -= ob.utxos[left].Amount
total += ob.utxos[right].Amount
Expand All @@ -858,8 +863,8 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u
results := make([]btcjson.ListUnspentResult, right-left)
copy(results, ob.utxos[left:right])

// Note: always put nonce-mark as 1st input
if idx >= 0 {
// include nonce-mark as the 1st input
if idx >= 0 { // for nonce > 0
if idx < left || idx >= right {
total += ob.utxos[idx].Amount
results = append([]btcjson.ListUnspentResult{ob.utxos[idx]}, results...)
Expand All @@ -870,9 +875,26 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u
}
}
if total < amount {
return nil, 0, fmt.Errorf("SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", total, amount)
return nil, 0, 0, 0, fmt.Errorf("SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", total, amount)
}

// consolidate biggest possible UTXOs to maximize consolidated value
// consolidation happens only when there are more than (or equal to) consolidateRank (10) UTXOs
utxoRank, consolidatedUtxo, consolidatedValue := uint16(0), uint16(0), 0.0
for i := len(ob.utxos) - 1; i >= 0 && utxosToSpend > 0; i-- { // iterate over UTXOs big-to-small
if i != idx && (i < left || i >= right) { // exclude nonce-mark and already selected UTXOs
utxoRank++
if utxoRank >= consolidateRank { // consolication starts from the 10-ranked UTXO based on value
utxosToSpend--
consolidatedUtxo++
total += ob.utxos[i].Amount
consolidatedValue += ob.utxos[i].Amount
results = append(results, ob.utxos[i])
}
}
}
return results, total, nil

return results, total, consolidatedUtxo, consolidatedValue, nil
}

// Save successfully broadcasted transaction
Expand Down
12 changes: 6 additions & 6 deletions zetaclient/btc_signer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ import (

const (
maxNoOfInputsPerTx = 20
outTxBytesMin = 400 // 500B is a conservative estimate for a 2-input, 3-output SegWit tx
outTxBytesMax = 4_000 // 4KB is a conservative estimate for a 21-input, 3-output SegWit tx
consolidationRank = 10 // the rank below (or equal to) which we consolidate UTXOs
outTxBytesMin = 400 // 500B is an estimated size for a 2-input, 3-output SegWit tx
outTxBytesMax = 3250 // 3250B is an estimated size for a 21-input, 3-output SegWit tx
outTxBytesCap = 10_000 // in case of accident

// for ZRC20 configuration
Expand Down Expand Up @@ -68,9 +69,7 @@ func NewBTCSigner(cfg config.BTCConfig, tssSigner TSSSigner, logger zerolog.Logg
// SignWithdrawTx receives utxos sorted by value, amount in BTC, feeRate in BTC per Kb
func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, amount float64, gasPrice *big.Int, sizeLimit uint64,
btcClient *BitcoinChainClient, height uint64, nonce uint64, chain *common.Chain) (*wire.MsgTx, error) {

estimateFee := float64(gasPrice.Uint64()) * outTxBytesMax / 1e8

nonceMark := common.NonceMarkAmount(nonce)

// refresh unspent UTXOs and continue with keysign regardless of error
Expand All @@ -80,7 +79,7 @@ func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, am
}

// select N UTXOs to cover the total expense
prevOuts, total, err := btcClient.SelectUTXOs(amount+estimateFee+float64(nonceMark)*1e-8, maxNoOfInputsPerTx, nonce, false)
prevOuts, total, consolidatedUtxo, consolidatedValue, err := btcClient.SelectUTXOs(amount+estimateFee+float64(nonceMark)*1e-8, maxNoOfInputsPerTx, nonce, consolidationRank, false)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -120,7 +119,8 @@ func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, am
// fee calculation
// #nosec G701 always in range (checked above)
fees := new(big.Int).Mul(big.NewInt(int64(txSize)), gasPrice)
signer.logger.Info().Msgf("bitcoin outTx nonce %d gasPrice %s size %d fees %s", nonce, gasPrice.String(), txSize, fees.String())
signer.logger.Info().Msgf("bitcoin outTx nonce %d gasPrice %s size %d fees %s consolidated %d utxos of value %v",
nonce, gasPrice.String(), txSize, fees.String(), consolidatedUtxo, consolidatedValue)

// calculate remaining btc to TSS self
tssAddrWPKH := signer.tssSigner.BTCAddressWitnessPubkeyHash()
Expand Down
172 changes: 135 additions & 37 deletions zetaclient/btc_signer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package zetaclient
import (
"encoding/hex"
"fmt"
"math"
"sort"
"sync"
"testing"
Expand Down Expand Up @@ -232,115 +233,212 @@ func mineTxNSetNonceMark(ob *BitcoinChainClient, nonce uint64, txid string, preM
ob.includedTxResults[outTxID] = btcjson.GetTransactionResult{TxID: txid}

// Set nonce mark
if preMarkIndex >= 0 {
tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress()
nonceMark := btcjson.ListUnspentResult{TxID: txid, Address: tssAddress, Amount: float64(common.NonceMarkAmount(nonce)) * 1e-8}
tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress()
nonceMark := btcjson.ListUnspentResult{TxID: txid, Address: tssAddress, Amount: float64(common.NonceMarkAmount(nonce)) * 1e-8}
if preMarkIndex >= 0 { // replace nonce-mark utxo
ob.utxos[preMarkIndex] = nonceMark
sort.SliceStable(ob.utxos, func(i, j int) bool {
return ob.utxos[i].Amount < ob.utxos[j].Amount
})

} else { // add nonce-mark utxo directly
ob.utxos = append(ob.utxos, nonceMark)
}
sort.SliceStable(ob.utxos, func(i, j int) bool {
return ob.utxos[i].Amount < ob.utxos[j].Amount
})
}

func TestSelectUTXOs(t *testing.T) {
ob := createTestClient(t)
tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress()
dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4"

// Case1: nonce = 0, bootstrap
// input: utxoCap = 5, amount = 0.01, nonce = 0
// output: [0.01], 0.01
result, amount, err := ob.SelectUTXOs(0.01, 5, 0, true)
result, amount, _, _, err := ob.SelectUTXOs(0.01, 5, 0, math.MaxUint16, true)
require.Nil(t, err)
require.Equal(t, 0.01, amount)
require.Equal(t, ob.utxos[0:1], result)

// Case2: nonce = 1, must FAIL and wait for previous transaction to be mined
// input: utxoCap = 5, amount = 0.5, nonce = 1
// output: error
result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true)
result, amount, _, _, err = ob.SelectUTXOs(0.5, 5, 1, math.MaxUint16, true)
require.NotNil(t, err)
require.Nil(t, result)
require.Zero(t, amount)
require.Equal(t, "getOutTxidByNonce: cannot find outTx txid for nonce 0", err.Error())
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction for nonce 0

// Case3: nonce = 1, must FAIL without nonce mark utxo
// input: utxoCap = 5, amount = 0.5, nonce = 1
// output: error
result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true)
require.NotNil(t, err)
require.Nil(t, result)
require.Zero(t, amount)
require.Equal(t, "findNonceMarkUTXO: cannot find nonce-mark utxo with nonce 0", err.Error())
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// add nonce-mark utxo for nonce 0
nonceMark0 := btcjson.ListUnspentResult{TxID: dummyTxID, Address: tssAddress, Amount: float64(common.NonceMarkAmount(0)) * 1e-8}
ob.utxos = append([]btcjson.ListUnspentResult{nonceMark0}, ob.utxos...)

// Case4: nonce = 1, should pass now
// Case3: nonce = 1, should pass now
// input: utxoCap = 5, amount = 0.5, nonce = 1
// output: [0.00002, 0.01, 0.12, 0.18, 0.24], 0.55002
result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true)
result, amount, _, _, err = ob.SelectUTXOs(0.5, 5, 1, math.MaxUint16, true)
require.Nil(t, err)
require.Equal(t, 0.55002, amount)
require.Equal(t, ob.utxos[0:5], result)
mineTxNSetNonceMark(ob, 1, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 1

// Case5:
// Case4:
// input: utxoCap = 5, amount = 1.0, nonce = 2
// output: [0.00002001, 0.01, 0.12, 0.18, 0.24, 0.5], 1.05002001
result, amount, err = ob.SelectUTXOs(1.0, 5, 2, true)
result, amount, _, _, err = ob.SelectUTXOs(1.0, 5, 2, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 1.05002001, amount, 1e-8)
require.Equal(t, ob.utxos[0:6], result)
mineTxNSetNonceMark(ob, 2, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 2

// Case6: should include nonce-mark utxo on the LEFT
// Case5: should include nonce-mark utxo on the LEFT
// input: utxoCap = 5, amount = 8.05, nonce = 3
// output: [0.00002002, 0.24, 0.5, 1.26, 2.97, 3.28], 8.25002002
result, amount, err = ob.SelectUTXOs(8.05, 5, 3, true)
result, amount, _, _, err = ob.SelectUTXOs(8.05, 5, 3, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 8.25002002, amount, 1e-8)
expected := append([]btcjson.ListUnspentResult{ob.utxos[0]}, ob.utxos[4:9]...)
require.Equal(t, expected, result)
mineTxNSetNonceMark(ob, 24105431, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 24105431

// Case7: should include nonce-mark utxo on the RIGHT
// Case6: should include nonce-mark utxo on the RIGHT
// input: utxoCap = 5, amount = 0.503, nonce = 24105432
// output: [0.24107432, 0.01, 0.12, 0.18, 0.24], 0.55002002
result, amount, err = ob.SelectUTXOs(0.503, 5, 24105432, true)
result, amount, _, _, err = ob.SelectUTXOs(0.503, 5, 24105432, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 0.79107431, amount, 1e-8)
expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:4]...)
require.Equal(t, expected, result)
mineTxNSetNonceMark(ob, 24105432, dummyTxID, 4) // mine a transaction and set nonce-mark utxo for nonce 24105432

// Case8: should include nonce-mark utxo in the MIDDLE
// Case7: should include nonce-mark utxo in the MIDDLE
// input: utxoCap = 5, amount = 1.0, nonce = 24105433
// output: [0.24107432, 0.12, 0.18, 0.24, 0.5], 1.28107432
result, amount, err = ob.SelectUTXOs(1.0, 5, 24105433, true)
result, amount, _, _, err = ob.SelectUTXOs(1.0, 5, 24105433, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 1.28107432, amount, 1e-8)
expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[1:4]...)
expected = append(expected, ob.utxos[5])
require.Equal(t, expected, result)

// Case9: should work with maximum amount
// Case8: should work with maximum amount
// input: utxoCap = 5, amount = 16.03
// output: [0.24107432, 1.26, 2.97, 3.28, 5.16, 8.72], 21.63107432
result, amount, err = ob.SelectUTXOs(16.03, 5, 24105433, true)
result, amount, _, _, err = ob.SelectUTXOs(16.03, 5, 24105433, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 21.63107432, amount, 1e-8)
expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[6:11]...)
require.Equal(t, expected, result)

// Case10: must FAIL due to insufficient funds
// Case9: must FAIL due to insufficient funds
// input: utxoCap = 5, amount = 21.64
// output: error
result, amount, err = ob.SelectUTXOs(21.64, 5, 24105433, true)
result, amount, _, _, err = ob.SelectUTXOs(21.64, 5, 24105433, math.MaxUint16, true)
require.NotNil(t, err)
require.Nil(t, result)
require.Zero(t, amount)
require.Equal(t, "SelectUTXOs: not enough btc in reserve - available : 21.63107432 , tx amount : 21.64", err.Error())
}

func TestUTXOConsolidation(t *testing.T) {
dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4"

t.Run("should not consolidate", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 10, amount = 0.01, nonce = 1, rank = 10
// output: [0.00002, 0.01], 0.01002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 10, 1, 10, true)
require.Nil(t, err)
require.Equal(t, 0.01002, amount)
require.Equal(t, ob.utxos[0:2], result)
require.Equal(t, uint16(0), clsdtUtxo)
require.Equal(t, 0.0, clsdtValue)
})

t.Run("should consolidate 1 utxo", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 9, amount = 0.01, nonce = 1, rank = 9
// output: [0.00002, 0.01, 0.12], 0.13002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 9, 1, 9, true)
require.Nil(t, err)
require.Equal(t, 0.13002, amount)
require.Equal(t, ob.utxos[0:3], result)
require.Equal(t, uint16(1), clsdtUtxo)
require.Equal(t, 0.12, clsdtValue)
})

t.Run("should consolidate 3 utxos", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 5, amount = 0.01, nonce = 0, rank = 5
// output: [0.00002, 0.014, 1.26, 0.5, 0.2], 2.01002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 5, 1, 5, true)
require.Nil(t, err)
require.Equal(t, 2.01002, amount)
expected := make([]btcjson.ListUnspentResult, 2)
copy(expected, ob.utxos[0:2])
for i := 6; i >= 4; i-- { // append consolidated utxos in descending order
expected = append(expected, ob.utxos[i])
}
require.Equal(t, expected, result)
require.Equal(t, uint16(3), clsdtUtxo)
require.Equal(t, 2.0, clsdtValue)
})

t.Run("should consolidate all utxos using rank 1", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 12, amount = 0.01, nonce = 0, rank = 1
// output: [0.00002, 0.01, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18, 0.12], 22.44002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 12, 1, 1, true)
require.Nil(t, err)
require.Equal(t, 22.44002, amount)
expected := make([]btcjson.ListUnspentResult, 2)
copy(expected, ob.utxos[0:2])
for i := 10; i >= 2; i-- { // append consolidated utxos in descending order
expected = append(expected, ob.utxos[i])
}
require.Equal(t, expected, result)
require.Equal(t, uint16(9), clsdtUtxo)
require.Equal(t, 22.43, clsdtValue)
})

t.Run("should consolidate 3 utxos sparse", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 24105431, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 24105431

// input: utxoCap = 5, amount = 0.13, nonce = 24105432, rank = 5
// output: [0.24107431, 0.01, 0.12, 1.26, 0.5, 0.24], 2.37107431
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.13, 5, 24105432, 5, true)
require.Nil(t, err)
assert.InEpsilon(t, 2.37107431, amount, 1e-8)
expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...)
expected = append(expected, ob.utxos[6])
expected = append(expected, ob.utxos[5])
expected = append(expected, ob.utxos[3])
require.Equal(t, expected, result)
require.Equal(t, uint16(3), clsdtUtxo)
require.Equal(t, 2.0, clsdtValue)
})

t.Run("should consolidate all utxos sparse", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 24105431, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 24105431

// input: utxoCap = 12, amount = 0.13, nonce = 24105432, rank = 1
// output: [0.24107431, 0.01, 0.12, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18], 22.68107431
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.13, 12, 24105432, 1, true)
require.Nil(t, err)
assert.InEpsilon(t, 22.68107431, amount, 1e-8)
expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...)
for i := 10; i >= 5; i-- { // append consolidated utxos in descending order
expected = append(expected, ob.utxos[i])
}
expected = append(expected, ob.utxos[3])
expected = append(expected, ob.utxos[2])
require.Equal(t, expected, result)
require.Equal(t, uint16(8), clsdtUtxo)
require.Equal(t, 22.31, clsdtValue)
})
}

0 comments on commit a4f2978

Please sign in to comment.