Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Bitcoin utxo consolidation #1326

Merged
merged 6 commits into from
Oct 23, 2023
Merged

feat: Bitcoin utxo consolidation #1326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e2cb0b8
initiated utxo consolidation
ws4charlie Oct 20, 2023
8daac18
use real world max outTx size
ws4charlie Oct 20, 2023
ec9b1bb
set consolidation rank to 10 to create a buffer (10) for small UTXOs …
ws4charlie Oct 20, 2023
618dd5e
revert DustOffset to 2000 for mainnet
ws4charlie Oct 20, 2023
2a137c4
improved variable names and comments
ws4charlie Oct 20, 2023
2919ba1
Merge branch 'develop' into feat-BTC-UTXO-consolidation
ws4charlie Oct 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion common/default_chains_mainnet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func BtcChainID() int64 {
}

func BtcDustOffset() int64 {
return 0
return 2000
}

func PolygonChain() Chain {
Expand Down
46 changes: 34 additions & 12 deletions zetaclient/bitcoin_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -815,12 +815,17 @@ func (ob *BitcoinChainClient) findNonceMarkUTXO(nonce uint64, txid string) (int,
//
// Parameters:
// - amount: The desired minimum total value of the selected UTXOs.
// - utxoCap: The maximum number of UTXOs to be selected.
// - utxos2Spend: The maximum number of UTXOs to spend.
// - nonce: The nonce of the outbound transaction.
// - consolidateRank: The rank below which UTXOs will be consolidated.
// - test: true for unit test only.
//
// Returns: a sublist (includes previous nonce-mark) of UTXOs or an error if the qulifying sublist cannot be found.
func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce uint64, test bool) ([]btcjson.ListUnspentResult, float64, error) {
// Returns:
// - a sublist (includes previous nonce-mark) of UTXOs or an error if the qualifying sublist cannot be found.
// - the total value of the selected UTXOs.
// - the number of consolidated UTXOs.
// - the total value of the consolidated UTXOs.
func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxosToSpend uint16, nonce uint64, consolidateRank uint16, test bool) ([]btcjson.ListUnspentResult, float64, uint16, float64, error) {
idx := -1
if nonce == 0 {
// for nonce = 0; make exception; no need to include nonce-mark utxo
Expand All @@ -830,24 +835,24 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u
// for nonce > 0; we proceed only when we see the nonce-mark utxo
preTxid, err := ob.getOutTxidByNonce(nonce-1, test)
if err != nil {
return nil, 0, err
return nil, 0, 0, 0, err
}
ob.Mu.Lock()
defer ob.Mu.Unlock()
idx, err = ob.findNonceMarkUTXO(nonce-1, preTxid)
if err != nil {
return nil, 0, err
return nil, 0, 0, 0, err
}
}

// select utxos
// select smallest possible UTXOs to make payment
total := 0.0
left, right := 0, 0
for total < amount && right < len(ob.utxos) {
if utxoCap > 0 { // expand sublist
if utxosToSpend > 0 { // expand sublist
total += ob.utxos[right].Amount
right++
utxoCap--
utxosToSpend--
} else { // pop the smallest utxo and append the current one
total -= ob.utxos[left].Amount
total += ob.utxos[right].Amount
Expand All @@ -858,8 +863,8 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u
results := make([]btcjson.ListUnspentResult, right-left)
copy(results, ob.utxos[left:right])

// Note: always put nonce-mark as 1st input
if idx >= 0 {
// include nonce-mark as the 1st input
if idx >= 0 { // for nonce > 0
if idx < left || idx >= right {
total += ob.utxos[idx].Amount
results = append([]btcjson.ListUnspentResult{ob.utxos[idx]}, results...)
Expand All @@ -870,9 +875,26 @@ func (ob *BitcoinChainClient) SelectUTXOs(amount float64, utxoCap uint8, nonce u
}
}
if total < amount {
return nil, 0, fmt.Errorf("SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", total, amount)
return nil, 0, 0, 0, fmt.Errorf("SelectUTXOs: not enough btc in reserve - available : %v , tx amount : %v", total, amount)
}

// consolidate biggest possible UTXOs to maximize consolidated value
// consolidation happens only when there are more than (or equal to) consolidateRank (10) UTXOs
utxoRank, consolidatedUtxo, consolidatedValue := uint16(0), uint16(0), 0.0
for i := len(ob.utxos) - 1; i >= 0 && utxosToSpend > 0; i-- { // iterate over UTXOs big-to-small
if i != idx && (i < left || i >= right) { // exclude nonce-mark and already selected UTXOs
utxoRank++
if utxoRank >= consolidateRank { // consolication starts from the 10-ranked UTXO based on value
utxosToSpend--
consolidatedUtxo++
total += ob.utxos[i].Amount
consolidatedValue += ob.utxos[i].Amount
results = append(results, ob.utxos[i])
}
}
}
return results, total, nil

return results, total, consolidatedUtxo, consolidatedValue, nil
}

// Save successfully broadcasted transaction
Expand Down
12 changes: 6 additions & 6 deletions zetaclient/btc_signer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ import (

const (
maxNoOfInputsPerTx = 20
outTxBytesMin = 400 // 500B is a conservative estimate for a 2-input, 3-output SegWit tx
outTxBytesMax = 4_000 // 4KB is a conservative estimate for a 21-input, 3-output SegWit tx
consolidationRank = 10 // the rank below (or equal to) which we consolidate UTXOs
outTxBytesMin = 400 // 500B is an estimated size for a 2-input, 3-output SegWit tx
outTxBytesMax = 3250 // 3250B is an estimated size for a 21-input, 3-output SegWit tx
outTxBytesCap = 10_000 // in case of accident

// for ZRC20 configuration
Expand Down Expand Up @@ -68,9 +69,7 @@ func NewBTCSigner(cfg config.BTCConfig, tssSigner TSSSigner, logger zerolog.Logg
// SignWithdrawTx receives utxos sorted by value, amount in BTC, feeRate in BTC per Kb
func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, amount float64, gasPrice *big.Int, sizeLimit uint64,
btcClient *BitcoinChainClient, height uint64, nonce uint64, chain *common.Chain) (*wire.MsgTx, error) {

estimateFee := float64(gasPrice.Uint64()) * outTxBytesMax / 1e8

nonceMark := common.NonceMarkAmount(nonce)

// refresh unspent UTXOs and continue with keysign regardless of error
Expand All @@ -80,7 +79,7 @@ func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, am
}

// select N UTXOs to cover the total expense
prevOuts, total, err := btcClient.SelectUTXOs(amount+estimateFee+float64(nonceMark)*1e-8, maxNoOfInputsPerTx, nonce, false)
prevOuts, total, consolidatedUtxo, consolidatedValue, err := btcClient.SelectUTXOs(amount+estimateFee+float64(nonceMark)*1e-8, maxNoOfInputsPerTx, nonce, consolidationRank, false)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -120,7 +119,8 @@ func (signer *BTCSigner) SignWithdrawTx(to *btcutil.AddressWitnessPubKeyHash, am
// fee calculation
// #nosec G701 always in range (checked above)
fees := new(big.Int).Mul(big.NewInt(int64(txSize)), gasPrice)
signer.logger.Info().Msgf("bitcoin outTx nonce %d gasPrice %s size %d fees %s", nonce, gasPrice.String(), txSize, fees.String())
signer.logger.Info().Msgf("bitcoin outTx nonce %d gasPrice %s size %d fees %s consolidated %d utxos of value %v",
nonce, gasPrice.String(), txSize, fees.String(), consolidatedUtxo, consolidatedValue)

// calculate remaining btc to TSS self
tssAddrWPKH := signer.tssSigner.BTCAddressWitnessPubkeyHash()
Expand Down
172 changes: 135 additions & 37 deletions zetaclient/btc_signer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package zetaclient
import (
"encoding/hex"
"fmt"
"math"
"sort"
"sync"
"testing"
Expand Down Expand Up @@ -232,115 +233,212 @@ func mineTxNSetNonceMark(ob *BitcoinChainClient, nonce uint64, txid string, preM
ob.includedTxResults[outTxID] = btcjson.GetTransactionResult{TxID: txid}

// Set nonce mark
if preMarkIndex >= 0 {
tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress()
nonceMark := btcjson.ListUnspentResult{TxID: txid, Address: tssAddress, Amount: float64(common.NonceMarkAmount(nonce)) * 1e-8}
tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress()
nonceMark := btcjson.ListUnspentResult{TxID: txid, Address: tssAddress, Amount: float64(common.NonceMarkAmount(nonce)) * 1e-8}
if preMarkIndex >= 0 { // replace nonce-mark utxo
ob.utxos[preMarkIndex] = nonceMark
sort.SliceStable(ob.utxos, func(i, j int) bool {
return ob.utxos[i].Amount < ob.utxos[j].Amount
})

} else { // add nonce-mark utxo directly
ob.utxos = append(ob.utxos, nonceMark)
}
sort.SliceStable(ob.utxos, func(i, j int) bool {
return ob.utxos[i].Amount < ob.utxos[j].Amount
})
}

func TestSelectUTXOs(t *testing.T) {
ob := createTestClient(t)
tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress()
dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4"

// Case1: nonce = 0, bootstrap
// input: utxoCap = 5, amount = 0.01, nonce = 0
// output: [0.01], 0.01
result, amount, err := ob.SelectUTXOs(0.01, 5, 0, true)
result, amount, _, _, err := ob.SelectUTXOs(0.01, 5, 0, math.MaxUint16, true)
require.Nil(t, err)
require.Equal(t, 0.01, amount)
require.Equal(t, ob.utxos[0:1], result)

// Case2: nonce = 1, must FAIL and wait for previous transaction to be mined
// input: utxoCap = 5, amount = 0.5, nonce = 1
// output: error
result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true)
result, amount, _, _, err = ob.SelectUTXOs(0.5, 5, 1, math.MaxUint16, true)
require.NotNil(t, err)
require.Nil(t, result)
require.Zero(t, amount)
require.Equal(t, "getOutTxidByNonce: cannot find outTx txid for nonce 0", err.Error())
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction for nonce 0

// Case3: nonce = 1, must FAIL without nonce mark utxo
// input: utxoCap = 5, amount = 0.5, nonce = 1
// output: error
result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true)
require.NotNil(t, err)
require.Nil(t, result)
require.Zero(t, amount)
require.Equal(t, "findNonceMarkUTXO: cannot find nonce-mark utxo with nonce 0", err.Error())
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// add nonce-mark utxo for nonce 0
nonceMark0 := btcjson.ListUnspentResult{TxID: dummyTxID, Address: tssAddress, Amount: float64(common.NonceMarkAmount(0)) * 1e-8}
ob.utxos = append([]btcjson.ListUnspentResult{nonceMark0}, ob.utxos...)

// Case4: nonce = 1, should pass now
// Case3: nonce = 1, should pass now
// input: utxoCap = 5, amount = 0.5, nonce = 1
// output: [0.00002, 0.01, 0.12, 0.18, 0.24], 0.55002
result, amount, err = ob.SelectUTXOs(0.5, 5, 1, true)
result, amount, _, _, err = ob.SelectUTXOs(0.5, 5, 1, math.MaxUint16, true)
require.Nil(t, err)
require.Equal(t, 0.55002, amount)
require.Equal(t, ob.utxos[0:5], result)
mineTxNSetNonceMark(ob, 1, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 1

// Case5:
// Case4:
// input: utxoCap = 5, amount = 1.0, nonce = 2
// output: [0.00002001, 0.01, 0.12, 0.18, 0.24, 0.5], 1.05002001
result, amount, err = ob.SelectUTXOs(1.0, 5, 2, true)
result, amount, _, _, err = ob.SelectUTXOs(1.0, 5, 2, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 1.05002001, amount, 1e-8)
require.Equal(t, ob.utxos[0:6], result)
mineTxNSetNonceMark(ob, 2, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 2

// Case6: should include nonce-mark utxo on the LEFT
// Case5: should include nonce-mark utxo on the LEFT
// input: utxoCap = 5, amount = 8.05, nonce = 3
// output: [0.00002002, 0.24, 0.5, 1.26, 2.97, 3.28], 8.25002002
result, amount, err = ob.SelectUTXOs(8.05, 5, 3, true)
result, amount, _, _, err = ob.SelectUTXOs(8.05, 5, 3, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 8.25002002, amount, 1e-8)
expected := append([]btcjson.ListUnspentResult{ob.utxos[0]}, ob.utxos[4:9]...)
require.Equal(t, expected, result)
mineTxNSetNonceMark(ob, 24105431, dummyTxID, 0) // mine a transaction and set nonce-mark utxo for nonce 24105431

// Case7: should include nonce-mark utxo on the RIGHT
// Case6: should include nonce-mark utxo on the RIGHT
// input: utxoCap = 5, amount = 0.503, nonce = 24105432
// output: [0.24107432, 0.01, 0.12, 0.18, 0.24], 0.55002002
result, amount, err = ob.SelectUTXOs(0.503, 5, 24105432, true)
result, amount, _, _, err = ob.SelectUTXOs(0.503, 5, 24105432, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 0.79107431, amount, 1e-8)
expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:4]...)
require.Equal(t, expected, result)
mineTxNSetNonceMark(ob, 24105432, dummyTxID, 4) // mine a transaction and set nonce-mark utxo for nonce 24105432

// Case8: should include nonce-mark utxo in the MIDDLE
// Case7: should include nonce-mark utxo in the MIDDLE
// input: utxoCap = 5, amount = 1.0, nonce = 24105433
// output: [0.24107432, 0.12, 0.18, 0.24, 0.5], 1.28107432
result, amount, err = ob.SelectUTXOs(1.0, 5, 24105433, true)
result, amount, _, _, err = ob.SelectUTXOs(1.0, 5, 24105433, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 1.28107432, amount, 1e-8)
expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[1:4]...)
expected = append(expected, ob.utxos[5])
require.Equal(t, expected, result)

// Case9: should work with maximum amount
// Case8: should work with maximum amount
// input: utxoCap = 5, amount = 16.03
// output: [0.24107432, 1.26, 2.97, 3.28, 5.16, 8.72], 21.63107432
result, amount, err = ob.SelectUTXOs(16.03, 5, 24105433, true)
result, amount, _, _, err = ob.SelectUTXOs(16.03, 5, 24105433, math.MaxUint16, true)
require.Nil(t, err)
assert.InEpsilon(t, 21.63107432, amount, 1e-8)
expected = append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[6:11]...)
require.Equal(t, expected, result)

// Case10: must FAIL due to insufficient funds
// Case9: must FAIL due to insufficient funds
// input: utxoCap = 5, amount = 21.64
// output: error
result, amount, err = ob.SelectUTXOs(21.64, 5, 24105433, true)
result, amount, _, _, err = ob.SelectUTXOs(21.64, 5, 24105433, math.MaxUint16, true)
require.NotNil(t, err)
require.Nil(t, result)
require.Zero(t, amount)
require.Equal(t, "SelectUTXOs: not enough btc in reserve - available : 21.63107432 , tx amount : 21.64", err.Error())
}

func TestUTXOConsolidation(t *testing.T) {
dummyTxID := "6e6f71d281146c1fc5c755b35908ee449f26786c84e2ae18f98b268de40b7ec4"

t.Run("should not consolidate", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 10, amount = 0.01, nonce = 1, rank = 10
// output: [0.00002, 0.01], 0.01002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 10, 1, 10, true)
require.Nil(t, err)
require.Equal(t, 0.01002, amount)
require.Equal(t, ob.utxos[0:2], result)
require.Equal(t, uint16(0), clsdtUtxo)
require.Equal(t, 0.0, clsdtValue)
})

t.Run("should consolidate 1 utxo", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 9, amount = 0.01, nonce = 1, rank = 9
// output: [0.00002, 0.01, 0.12], 0.13002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 9, 1, 9, true)
require.Nil(t, err)
require.Equal(t, 0.13002, amount)
require.Equal(t, ob.utxos[0:3], result)
require.Equal(t, uint16(1), clsdtUtxo)
require.Equal(t, 0.12, clsdtValue)
})

t.Run("should consolidate 3 utxos", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 5, amount = 0.01, nonce = 0, rank = 5
// output: [0.00002, 0.014, 1.26, 0.5, 0.2], 2.01002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 5, 1, 5, true)
require.Nil(t, err)
require.Equal(t, 2.01002, amount)
expected := make([]btcjson.ListUnspentResult, 2)
copy(expected, ob.utxos[0:2])
for i := 6; i >= 4; i-- { // append consolidated utxos in descending order
expected = append(expected, ob.utxos[i])
}
require.Equal(t, expected, result)
require.Equal(t, uint16(3), clsdtUtxo)
require.Equal(t, 2.0, clsdtValue)
})

t.Run("should consolidate all utxos using rank 1", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 0, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 0

// input: utxoCap = 12, amount = 0.01, nonce = 0, rank = 1
// output: [0.00002, 0.01, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18, 0.12], 22.44002
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.01, 12, 1, 1, true)
require.Nil(t, err)
require.Equal(t, 22.44002, amount)
expected := make([]btcjson.ListUnspentResult, 2)
copy(expected, ob.utxos[0:2])
for i := 10; i >= 2; i-- { // append consolidated utxos in descending order
expected = append(expected, ob.utxos[i])
}
require.Equal(t, expected, result)
require.Equal(t, uint16(9), clsdtUtxo)
require.Equal(t, 22.43, clsdtValue)
})

t.Run("should consolidate 3 utxos sparse", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 24105431, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 24105431

// input: utxoCap = 5, amount = 0.13, nonce = 24105432, rank = 5
// output: [0.24107431, 0.01, 0.12, 1.26, 0.5, 0.24], 2.37107431
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.13, 5, 24105432, 5, true)
require.Nil(t, err)
assert.InEpsilon(t, 2.37107431, amount, 1e-8)
expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...)
expected = append(expected, ob.utxos[6])
expected = append(expected, ob.utxos[5])
expected = append(expected, ob.utxos[3])
require.Equal(t, expected, result)
require.Equal(t, uint16(3), clsdtUtxo)
require.Equal(t, 2.0, clsdtValue)
})

t.Run("should consolidate all utxos sparse", func(t *testing.T) {
ob := createTestClient(t)
mineTxNSetNonceMark(ob, 24105431, dummyTxID, -1) // mine a transaction and set nonce-mark utxo for nonce 24105431

// input: utxoCap = 12, amount = 0.13, nonce = 24105432, rank = 1
// output: [0.24107431, 0.01, 0.12, 8.72, 5.16, 3.28, 2.97, 1.26, 0.5, 0.24, 0.18], 22.68107431
result, amount, clsdtUtxo, clsdtValue, err := ob.SelectUTXOs(0.13, 12, 24105432, 1, true)
require.Nil(t, err)
assert.InEpsilon(t, 22.68107431, amount, 1e-8)
expected := append([]btcjson.ListUnspentResult{ob.utxos[4]}, ob.utxos[0:2]...)
for i := 10; i >= 5; i-- { // append consolidated utxos in descending order
expected = append(expected, ob.utxos[i])
}
expected = append(expected, ob.utxos[3])
expected = append(expected, ob.utxos[2])
require.Equal(t, expected, result)
require.Equal(t, uint16(8), clsdtUtxo)
require.Equal(t, 22.31, clsdtValue)
})
}
Loading