From bdf345732dcb3118375aa0209067d5961f3d944d Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 28 Nov 2024 19:33:44 -0300 Subject: [PATCH 01/31] track status on each signature to detect reorgs --- pkg/solana/txm/pendingtx.go | 163 ++++++++++++++++--------------- pkg/solana/txm/pendingtx_test.go | 46 ++++----- 2 files changed, 107 insertions(+), 102 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 44895fd40..f5cd214b4 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -66,11 +66,16 @@ type finishedTx struct { state TxState } +type txInfo struct { + id string + status TxState +} + var _ PendingTxContext = &pendingTxContext{} type pendingTxContext struct { - cancelBy map[string]context.CancelFunc - sigToID map[solana.Signature]string + cancelBy map[string]context.CancelFunc + sigToTxInfo map[solana.Signature]txInfo broadcastedProcessedTxs map[string]pendingTx // broadcasted and processed transactions that may require retry and bumping confirmedTxs map[string]pendingTx // transactions that require monitoring for re-org @@ -81,8 +86,8 @@ type pendingTxContext struct { func newPendingTxContext() *pendingTxContext { return &pendingTxContext{ - cancelBy: map[string]context.CancelFunc{}, - sigToID: map[solana.Signature]string{}, + cancelBy: map[string]context.CancelFunc{}, + sigToTxInfo: map[solana.Signature]txInfo{}, broadcastedProcessedTxs: map[string]pendingTx{}, confirmedTxs: map[string]pendingTx{}, @@ -93,7 +98,7 @@ func newPendingTxContext() *pendingTxContext { func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel context.CancelFunc) error { err := c.withReadLock(func() error { // validate signature does not exist - if _, exists := c.sigToID[sig]; exists { + if _, exists := c.sigToTxInfo[sig]; exists { return ErrSigAlreadyExists } // validate id does not exist @@ -108,7 +113,7 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex // upgrade to write lock if sig or id do not exist _, err = c.withWriteLock(func() (string, error) { - if _, exists := c.sigToID[sig]; exists { + if _, exists := c.sigToTxInfo[sig]; exists { return "", ErrSigAlreadyExists } if _, exists := c.broadcastedProcessedTxs[tx.id]; exists { @@ -116,7 +121,7 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex } // save cancel func c.cancelBy[tx.id] = cancel - c.sigToID[sig] = tx.id + c.sigToTxInfo[sig] = txInfo{id: tx.id, status: Broadcasted} // add signature to tx tx.signatures = append(tx.signatures, sig) tx.createTs = time.Now() @@ -131,7 +136,7 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { err := c.withReadLock(func() error { // signature already exists - if _, exists := c.sigToID[sig]; exists { + if _, exists := c.sigToTxInfo[sig]; exists { return ErrSigAlreadyExists } // new signatures should only be added for broadcasted transactions @@ -147,13 +152,13 @@ func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { // upgrade to write lock if sig does not exist _, err = c.withWriteLock(func() (string, error) { - if _, exists := c.sigToID[sig]; exists { + if _, exists := c.sigToTxInfo[sig]; exists { return "", ErrSigAlreadyExists } if _, exists := c.broadcastedProcessedTxs[id]; !exists { return "", ErrTransactionNotFound } - c.sigToID[sig] = id + c.sigToTxInfo[sig] = txInfo{id: id, status: Broadcasted} tx := c.broadcastedProcessedTxs[id] // save new signature tx.signatures = append(tx.signatures, sig) @@ -169,12 +174,12 @@ func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { func (c *pendingTxContext) Remove(sig solana.Signature) (id string, err error) { err = c.withReadLock(func() error { // check if already removed - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } - _, broadcastedIDExists := c.broadcastedProcessedTxs[id] - _, confirmedIDExists := c.confirmedTxs[id] + _, broadcastedIDExists := c.broadcastedProcessedTxs[txInfo.id] + _, confirmedIDExists := c.confirmedTxs[txInfo.id] // transcation does not exist in tx maps if !broadcastedIDExists && !confirmedIDExists { return ErrTransactionNotFound @@ -187,38 +192,38 @@ func (c *pendingTxContext) Remove(sig solana.Signature) (id string, err error) { // upgrade to write lock if sig does not exist return c.withWriteLock(func() (string, error) { - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { - return id, ErrSigDoesNotExist + return txInfo.id, ErrSigDoesNotExist } var tx pendingTx - if tempTx, exists := c.broadcastedProcessedTxs[id]; exists { + if tempTx, exists := c.broadcastedProcessedTxs[txInfo.id]; exists { tx = tempTx - delete(c.broadcastedProcessedTxs, id) + delete(c.broadcastedProcessedTxs, txInfo.id) } - if tempTx, exists := c.confirmedTxs[id]; exists { + if tempTx, exists := c.confirmedTxs[txInfo.id]; exists { tx = tempTx - delete(c.confirmedTxs, id) + delete(c.confirmedTxs, txInfo.id) } // call cancel func + remove from map - if cancel, exists := c.cancelBy[id]; exists { + if cancel, exists := c.cancelBy[txInfo.id]; exists { cancel() // cancel context - delete(c.cancelBy, id) + delete(c.cancelBy, txInfo.id) } // remove all signatures associated with transaction from sig map for _, s := range tx.signatures { - delete(c.sigToID, s) + delete(c.sigToTxInfo, s) } - return id, nil + return txInfo.id, nil }) } func (c *pendingTxContext) ListAll() []solana.Signature { c.lock.RLock() defer c.lock.RUnlock() - return maps.Keys(c.sigToID) + return maps.Keys(c.sigToTxInfo) } // ListAllExpiredBroadcastedTxs returns all the txes that are in broadcasted state and have expired for given slot height compared against their lastValidBlockHeight. @@ -243,14 +248,14 @@ func (c *pendingTxContext) Expired(sig solana.Signature, confirmationTimeout tim if confirmationTimeout == 0 { return false } - id, exists := c.sigToID[sig] + txInfo, exists := c.sigToTxInfo[sig] if !exists { return false // return expired = false if timestamp does not exist (likely cleaned up by something else previously) } - if tx, exists := c.broadcastedProcessedTxs[id]; exists { + if tx, exists := c.broadcastedProcessedTxs[txInfo.id]; exists { return time.Since(tx.createTs) > confirmationTimeout } - if tx, exists := c.confirmedTxs[id]; exists { + if tx, exists := c.confirmedTxs[txInfo.id]; exists { return time.Since(tx.createTs) > confirmationTimeout } return false // return expired = false if tx does not exist (likely cleaned up by something else previously) @@ -259,12 +264,12 @@ func (c *pendingTxContext) Expired(sig solana.Signature, confirmationTimeout tim func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { err := c.withReadLock(func() error { // validate if sig exists - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // Transactions should only move to processed from broadcasted - tx, exists := c.broadcastedProcessedTxs[id] + tx, exists := c.broadcastedProcessedTxs[txInfo.id] if !exists { return ErrTransactionNotFound } @@ -280,35 +285,35 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { // upgrade to write lock if sig and id exist return c.withWriteLock(func() (string, error) { - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { - return id, ErrSigDoesNotExist + return txInfo.id, ErrSigDoesNotExist } - tx, exists := c.broadcastedProcessedTxs[id] + tx, exists := c.broadcastedProcessedTxs[txInfo.id] if !exists { - return id, ErrTransactionNotFound + return txInfo.id, ErrTransactionNotFound } // update tx state to Processed tx.state = Processed // save updated tx back to the broadcasted map - c.broadcastedProcessedTxs[id] = tx - return id, nil + c.broadcastedProcessedTxs[txInfo.id] = tx + return txInfo.id, nil }) } func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { err := c.withReadLock(func() error { // validate if sig exists - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // Check if transaction already in confirmed state - if tx, exists := c.confirmedTxs[id]; exists && tx.state == Confirmed { + if tx, exists := c.confirmedTxs[txInfo.id]; exists && tx.state == Confirmed { return ErrAlreadyInExpectedState } // Transactions should only move to confirmed from broadcasted/processed - if _, exists := c.broadcastedProcessedTxs[id]; !exists { + if _, exists := c.broadcastedProcessedTxs[txInfo.id]; !exists { return ErrTransactionNotFound } return nil @@ -319,38 +324,38 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { // upgrade to write lock if id exists return c.withWriteLock(func() (string, error) { - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { - return id, ErrSigDoesNotExist + return txInfo.id, ErrSigDoesNotExist } - tx, exists := c.broadcastedProcessedTxs[id] + tx, exists := c.broadcastedProcessedTxs[txInfo.id] if !exists { - return id, ErrTransactionNotFound + return txInfo.id, ErrTransactionNotFound } // call cancel func + remove from map to stop the retry/bumping cycle for this transaction - if cancel, exists := c.cancelBy[id]; exists { + if cancel, exists := c.cancelBy[txInfo.id]; exists { cancel() // cancel context - delete(c.cancelBy, id) + delete(c.cancelBy, txInfo.id) } // update tx state to Confirmed tx.state = Confirmed // move tx to confirmed map - c.confirmedTxs[id] = tx + c.confirmedTxs[txInfo.id] = tx // remove tx from broadcasted map - delete(c.broadcastedProcessedTxs, id) - return id, nil + delete(c.broadcastedProcessedTxs, txInfo.id) + return txInfo.id, nil }) } func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout time.Duration) (string, error) { err := c.withReadLock(func() error { - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // Allow transactions to transition from broadcasted, processed, or confirmed state in case there are delays between status checks - _, broadcastedExists := c.broadcastedProcessedTxs[id] - _, confirmedExists := c.confirmedTxs[id] + _, broadcastedExists := c.broadcastedProcessedTxs[txInfo.id] + _, confirmedExists := c.confirmedTxs[txInfo.id] if !broadcastedExists && !confirmedExists { return ErrTransactionNotFound } @@ -362,47 +367,47 @@ func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout ti // upgrade to write lock if id exists return c.withWriteLock(func() (string, error) { - id, exists := c.sigToID[sig] + txInfo, exists := c.sigToTxInfo[sig] if !exists { - return id, ErrSigDoesNotExist + return txInfo.id, ErrSigDoesNotExist } var tx, tempTx pendingTx var broadcastedExists, confirmedExists bool - if tempTx, broadcastedExists = c.broadcastedProcessedTxs[id]; broadcastedExists { + if tempTx, broadcastedExists = c.broadcastedProcessedTxs[txInfo.id]; broadcastedExists { tx = tempTx } - if tempTx, confirmedExists = c.confirmedTxs[id]; confirmedExists { + if tempTx, confirmedExists = c.confirmedTxs[txInfo.id]; confirmedExists { tx = tempTx } if !broadcastedExists && !confirmedExists { - return id, ErrTransactionNotFound + return txInfo.id, ErrTransactionNotFound } // call cancel func + remove from map to stop the retry/bumping cycle for this transaction // cancel is expected to be called and removed when tx is confirmed but checked here too in case state is skipped - if cancel, exists := c.cancelBy[id]; exists { + if cancel, exists := c.cancelBy[txInfo.id]; exists { cancel() // cancel context - delete(c.cancelBy, id) + delete(c.cancelBy, txInfo.id) } // delete from broadcasted map, if exists - delete(c.broadcastedProcessedTxs, id) + delete(c.broadcastedProcessedTxs, txInfo.id) // delete from confirmed map, if exists - delete(c.confirmedTxs, id) - // remove all related signatures from the sigToID map to skip picking up this tx in the confirmation logic + delete(c.confirmedTxs, txInfo.id) + // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic for _, s := range tx.signatures { - delete(c.sigToID, s) + delete(c.sigToTxInfo, s) } // if retention duration is set to 0, delete transaction from storage // otherwise, move to finalized map if retentionTimeout == 0 { - return id, nil + return txInfo.id, nil } finalizedTx := finishedTx{ state: Finalized, retentionTs: time.Now().Add(retentionTimeout), } // move transaction from confirmed to finalized map - c.finalizedErroredTxs[id] = finalizedTx - return id, nil + c.finalizedErroredTxs[txInfo.id] = finalizedTx + return txInfo.id, nil }) } @@ -450,14 +455,14 @@ func (c *pendingTxContext) OnPrebroadcastError(id string, retentionTimeout time. func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.Duration, txState TxState, _ TxErrType) (string, error) { err := c.withReadLock(func() error { - id, sigExists := c.sigToID[sig] + txInfo, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // transaction can transition from any non-finalized state var broadcastedExists, confirmedExists bool - _, broadcastedExists = c.broadcastedProcessedTxs[id] - _, confirmedExists = c.confirmedTxs[id] + _, broadcastedExists = c.broadcastedProcessedTxs[txInfo.id] + _, confirmedExists = c.confirmedTxs[txInfo.id] // transcation does not exist in any tx maps if !broadcastedExists && !confirmedExists { return ErrTransactionNotFound @@ -470,16 +475,16 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D // upgrade to write lock if sig exists return c.withWriteLock(func() (string, error) { - id, exists := c.sigToID[sig] + txInfo, exists := c.sigToTxInfo[sig] if !exists { return "", ErrSigDoesNotExist } var tx, tempTx pendingTx var broadcastedExists, confirmedExists bool - if tempTx, broadcastedExists = c.broadcastedProcessedTxs[id]; broadcastedExists { + if tempTx, broadcastedExists = c.broadcastedProcessedTxs[txInfo.id]; broadcastedExists { tx = tempTx } - if tempTx, confirmedExists = c.confirmedTxs[id]; confirmedExists { + if tempTx, confirmedExists = c.confirmedTxs[txInfo.id]; confirmedExists { tx = tempTx } // transcation does not exist in any non-finalized maps @@ -487,29 +492,29 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D return "", ErrTransactionNotFound } // call cancel func + remove from map - if cancel, exists := c.cancelBy[id]; exists { + if cancel, exists := c.cancelBy[txInfo.id]; exists { cancel() // cancel context - delete(c.cancelBy, id) + delete(c.cancelBy, txInfo.id) } // delete from broadcasted map, if exists - delete(c.broadcastedProcessedTxs, id) + delete(c.broadcastedProcessedTxs, txInfo.id) // delete from confirmed map, if exists - delete(c.confirmedTxs, id) - // remove all related signatures from the sigToID map to skip picking up this tx in the confirmation logic + delete(c.confirmedTxs, txInfo.id) + // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic for _, s := range tx.signatures { - delete(c.sigToID, s) + delete(c.sigToTxInfo, s) } // if retention duration is set to 0, skip adding transaction to the errored map if retentionTimeout == 0 { - return id, nil + return txInfo.id, nil } erroredTx := finishedTx{ state: txState, retentionTs: time.Now().Add(retentionTimeout), } // move transaction from broadcasted to error map - c.finalizedErroredTxs[id] = erroredTx - return id, nil + c.finalizedErroredTxs[txInfo.id] = erroredTx + return txInfo.id, nil }) } diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 472651f26..60b208412 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -79,9 +79,9 @@ func TestPendingTxContext_new(t *testing.T) { require.NoError(t, err) // Check it exists in signature map - id, exists := txs.sigToID[sig] + txInfo, exists := txs.sigToTxInfo[sig] require.True(t, exists) - require.Equal(t, msg.id, id) + require.Equal(t, msg.id, txInfo.id) // Check it exists in broadcasted map tx, exists := txs.broadcastedProcessedTxs[msg.id] @@ -119,12 +119,12 @@ func TestPendingTxContext_add_signature(t *testing.T) { require.NoError(t, err) // Check signature map - id, exists := txs.sigToID[sig1] + txInfo, exists := txs.sigToTxInfo[sig1] require.True(t, exists) - require.Equal(t, msg.id, id) - id, exists = txs.sigToID[sig2] + require.Equal(t, msg.id, txInfo.id) + txInfo, exists = txs.sigToTxInfo[sig2] require.True(t, exists) - require.Equal(t, msg.id, id) + require.Equal(t, msg.id, txInfo.id) // Check broadcasted map tx, exists := txs.broadcastedProcessedTxs[msg.id] @@ -211,9 +211,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { require.Equal(t, msg.id, id) // Check it exists in signature map - id, exists := txs.sigToID[sig] + txInfo, exists := txs.sigToTxInfo[sig] require.True(t, exists) - require.Equal(t, msg.id, id) + require.Equal(t, msg.id, txInfo.id) // Check it exists in broadcasted map tx, exists := txs.broadcastedProcessedTxs[msg.id] @@ -346,9 +346,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { require.Equal(t, msg.id, id) // Check it exists in signature map - id, exists := txs.sigToID[sig] + txInfo, exists := txs.sigToTxInfo[sig] require.True(t, exists) - require.Equal(t, msg.id, id) + require.Equal(t, msg.id, txInfo.id) // Check it does not exist in broadcasted map _, exists = txs.broadcastedProcessedTxs[msg.id] @@ -478,9 +478,9 @@ func TestPendingTxContext_on_finalized(t *testing.T) { require.Equal(t, Finalized, tx.state) // Check sigs do no exist in signature map - _, exists = txs.sigToID[sig1] + _, exists = txs.sigToTxInfo[sig1] require.False(t, exists) - _, exists = txs.sigToID[sig2] + _, exists = txs.sigToTxInfo[sig2] require.False(t, exists) }) @@ -528,9 +528,9 @@ func TestPendingTxContext_on_finalized(t *testing.T) { require.Equal(t, Finalized, tx.state) // Check sigs do no exist in signature map - _, exists = txs.sigToID[sig1] + _, exists = txs.sigToTxInfo[sig1] require.False(t, exists) - _, exists = txs.sigToID[sig2] + _, exists = txs.sigToTxInfo[sig2] require.False(t, exists) }) @@ -570,7 +570,7 @@ func TestPendingTxContext_on_finalized(t *testing.T) { require.False(t, exists) // Check sigs do no exist in signature map - _, exists = txs.sigToID[sig1] + _, exists = txs.sigToTxInfo[sig1] require.False(t, exists) }) @@ -628,7 +628,7 @@ func TestPendingTxContext_on_error(t *testing.T) { require.Equal(t, Errored, tx.state) // Check sigs do no exist in signature map - _, exists = txs.sigToID[sig] + _, exists = txs.sigToTxInfo[sig] require.False(t, exists) }) @@ -666,7 +666,7 @@ func TestPendingTxContext_on_error(t *testing.T) { require.Equal(t, Errored, tx.state) // Check sigs do no exist in signature map - _, exists = txs.sigToID[sig] + _, exists = txs.sigToTxInfo[sig] require.False(t, exists) }) @@ -695,7 +695,7 @@ func TestPendingTxContext_on_error(t *testing.T) { require.Equal(t, FatallyErrored, tx.state) // Check sigs do no exist in signature map - _, exists = txs.sigToID[sig] + _, exists = txs.sigToTxInfo[sig] require.False(t, exists) }) @@ -730,7 +730,7 @@ func TestPendingTxContext_on_error(t *testing.T) { require.False(t, exists) // Check sigs do no exist in signature map - _, exists = txs.sigToID[sig] + _, exists = txs.sigToTxInfo[sig] require.False(t, exists) }) @@ -879,9 +879,9 @@ func TestPendingTxContext_remove(t *testing.T) { _, exists := txs.broadcastedProcessedTxs[broadcastedMsg.id] require.False(t, exists) // Check all signatures removed from sig map - _, exists = txs.sigToID[broadcastedSig1] + _, exists = txs.sigToTxInfo[broadcastedSig1] require.False(t, exists) - _, exists = txs.sigToID[broadcastedSig2] + _, exists = txs.sigToTxInfo[broadcastedSig2] require.False(t, exists) // Remove processed transaction @@ -892,7 +892,7 @@ func TestPendingTxContext_remove(t *testing.T) { _, exists = txs.broadcastedProcessedTxs[processedMsg.id] require.False(t, exists) // Check all signatures removed from sig map - _, exists = txs.sigToID[processedSig] + _, exists = txs.sigToTxInfo[processedSig] require.False(t, exists) // Remove confirmed transaction @@ -903,7 +903,7 @@ func TestPendingTxContext_remove(t *testing.T) { _, exists = txs.confirmedTxs[confirmedMsg.id] require.False(t, exists) // Check all signatures removed from sig map - _, exists = txs.sigToID[confirmedSig] + _, exists = txs.sigToTxInfo[confirmedSig] require.False(t, exists) // Check remove cannot be called on finalized transaction From 3ad2bc831a788f6b44c47b7fa8b87f6da4a4c595 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Fri, 29 Nov 2024 12:55:12 -0300 Subject: [PATCH 02/31] move things arround + add reorg detection --- pkg/solana/txm/pendingtx.go | 245 ++++++++++++++++++++++++++---------- pkg/solana/txm/txm.go | 60 +++++++-- pkg/solana/txm/utils.go | 28 +++++ 3 files changed, 261 insertions(+), 72 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index f5cd214b4..0f8d5bef1 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -47,6 +47,12 @@ type PendingTxContext interface { GetTxState(id string) (TxState, error) // TrimFinalizedErroredTxs removes transactions that have reached their retention time TrimFinalizedErroredTxs() int + // GetSignatureInfo returns the transaction ID and TxState for the provided signature + GetSignatureInfo(sig solana.Signature) (txInfo, error) + // UpdateSignatureStatus updates the status of the provided signature within sigToTxInfo map + UpdateSignatureStatus(sig solana.Signature, newStatus TxState) (string, error) + // OnReorg resets the transaction state to Broadcasted for the given signature and returns the pendingTx. + OnReorg(sig solana.Signature) (pendingTx, error) } // finishedTx is used to store info required to track transactions to finality or error @@ -174,12 +180,12 @@ func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { func (c *pendingTxContext) Remove(sig solana.Signature) (id string, err error) { err = c.withReadLock(func() error { // check if already removed - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } - _, broadcastedIDExists := c.broadcastedProcessedTxs[txInfo.id] - _, confirmedIDExists := c.confirmedTxs[txInfo.id] + _, broadcastedIDExists := c.broadcastedProcessedTxs[info.id] + _, confirmedIDExists := c.confirmedTxs[info.id] // transcation does not exist in tx maps if !broadcastedIDExists && !confirmedIDExists { return ErrTransactionNotFound @@ -192,31 +198,31 @@ func (c *pendingTxContext) Remove(sig solana.Signature) (id string, err error) { // upgrade to write lock if sig does not exist return c.withWriteLock(func() (string, error) { - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { - return txInfo.id, ErrSigDoesNotExist + return info.id, ErrSigDoesNotExist } var tx pendingTx - if tempTx, exists := c.broadcastedProcessedTxs[txInfo.id]; exists { + if tempTx, exists := c.broadcastedProcessedTxs[info.id]; exists { tx = tempTx - delete(c.broadcastedProcessedTxs, txInfo.id) + delete(c.broadcastedProcessedTxs, info.id) } - if tempTx, exists := c.confirmedTxs[txInfo.id]; exists { + if tempTx, exists := c.confirmedTxs[info.id]; exists { tx = tempTx - delete(c.confirmedTxs, txInfo.id) + delete(c.confirmedTxs, info.id) } // call cancel func + remove from map - if cancel, exists := c.cancelBy[txInfo.id]; exists { + if cancel, exists := c.cancelBy[info.id]; exists { cancel() // cancel context - delete(c.cancelBy, txInfo.id) + delete(c.cancelBy, info.id) } // remove all signatures associated with transaction from sig map for _, s := range tx.signatures { delete(c.sigToTxInfo, s) } - return txInfo.id, nil + return info.id, nil }) } @@ -248,14 +254,14 @@ func (c *pendingTxContext) Expired(sig solana.Signature, confirmationTimeout tim if confirmationTimeout == 0 { return false } - txInfo, exists := c.sigToTxInfo[sig] + info, exists := c.sigToTxInfo[sig] if !exists { return false // return expired = false if timestamp does not exist (likely cleaned up by something else previously) } - if tx, exists := c.broadcastedProcessedTxs[txInfo.id]; exists { + if tx, exists := c.broadcastedProcessedTxs[info.id]; exists { return time.Since(tx.createTs) > confirmationTimeout } - if tx, exists := c.confirmedTxs[txInfo.id]; exists { + if tx, exists := c.confirmedTxs[info.id]; exists { return time.Since(tx.createTs) > confirmationTimeout } return false // return expired = false if tx does not exist (likely cleaned up by something else previously) @@ -264,12 +270,12 @@ func (c *pendingTxContext) Expired(sig solana.Signature, confirmationTimeout tim func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { err := c.withReadLock(func() error { // validate if sig exists - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // Transactions should only move to processed from broadcasted - tx, exists := c.broadcastedProcessedTxs[txInfo.id] + tx, exists := c.broadcastedProcessedTxs[info.id] if !exists { return ErrTransactionNotFound } @@ -285,35 +291,35 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { // upgrade to write lock if sig and id exist return c.withWriteLock(func() (string, error) { - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { - return txInfo.id, ErrSigDoesNotExist + return info.id, ErrSigDoesNotExist } - tx, exists := c.broadcastedProcessedTxs[txInfo.id] + tx, exists := c.broadcastedProcessedTxs[info.id] if !exists { - return txInfo.id, ErrTransactionNotFound + return info.id, ErrTransactionNotFound } // update tx state to Processed tx.state = Processed // save updated tx back to the broadcasted map - c.broadcastedProcessedTxs[txInfo.id] = tx - return txInfo.id, nil + c.broadcastedProcessedTxs[info.id] = tx + return info.id, nil }) } func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { err := c.withReadLock(func() error { // validate if sig exists - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // Check if transaction already in confirmed state - if tx, exists := c.confirmedTxs[txInfo.id]; exists && tx.state == Confirmed { + if tx, exists := c.confirmedTxs[info.id]; exists && tx.state == Confirmed { return ErrAlreadyInExpectedState } // Transactions should only move to confirmed from broadcasted/processed - if _, exists := c.broadcastedProcessedTxs[txInfo.id]; !exists { + if _, exists := c.broadcastedProcessedTxs[info.id]; !exists { return ErrTransactionNotFound } return nil @@ -324,38 +330,38 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { // upgrade to write lock if id exists return c.withWriteLock(func() (string, error) { - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { - return txInfo.id, ErrSigDoesNotExist + return info.id, ErrSigDoesNotExist } - tx, exists := c.broadcastedProcessedTxs[txInfo.id] + tx, exists := c.broadcastedProcessedTxs[info.id] if !exists { - return txInfo.id, ErrTransactionNotFound + return info.id, ErrTransactionNotFound } // call cancel func + remove from map to stop the retry/bumping cycle for this transaction - if cancel, exists := c.cancelBy[txInfo.id]; exists { + if cancel, exists := c.cancelBy[info.id]; exists { cancel() // cancel context - delete(c.cancelBy, txInfo.id) + delete(c.cancelBy, info.id) } // update tx state to Confirmed tx.state = Confirmed // move tx to confirmed map - c.confirmedTxs[txInfo.id] = tx + c.confirmedTxs[info.id] = tx // remove tx from broadcasted map - delete(c.broadcastedProcessedTxs, txInfo.id) - return txInfo.id, nil + delete(c.broadcastedProcessedTxs, info.id) + return info.id, nil }) } func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout time.Duration) (string, error) { err := c.withReadLock(func() error { - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // Allow transactions to transition from broadcasted, processed, or confirmed state in case there are delays between status checks - _, broadcastedExists := c.broadcastedProcessedTxs[txInfo.id] - _, confirmedExists := c.confirmedTxs[txInfo.id] + _, broadcastedExists := c.broadcastedProcessedTxs[info.id] + _, confirmedExists := c.confirmedTxs[info.id] if !broadcastedExists && !confirmedExists { return ErrTransactionNotFound } @@ -367,31 +373,31 @@ func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout ti // upgrade to write lock if id exists return c.withWriteLock(func() (string, error) { - txInfo, exists := c.sigToTxInfo[sig] + info, exists := c.sigToTxInfo[sig] if !exists { - return txInfo.id, ErrSigDoesNotExist + return info.id, ErrSigDoesNotExist } var tx, tempTx pendingTx var broadcastedExists, confirmedExists bool - if tempTx, broadcastedExists = c.broadcastedProcessedTxs[txInfo.id]; broadcastedExists { + if tempTx, broadcastedExists = c.broadcastedProcessedTxs[info.id]; broadcastedExists { tx = tempTx } - if tempTx, confirmedExists = c.confirmedTxs[txInfo.id]; confirmedExists { + if tempTx, confirmedExists = c.confirmedTxs[info.id]; confirmedExists { tx = tempTx } if !broadcastedExists && !confirmedExists { - return txInfo.id, ErrTransactionNotFound + return info.id, ErrTransactionNotFound } // call cancel func + remove from map to stop the retry/bumping cycle for this transaction // cancel is expected to be called and removed when tx is confirmed but checked here too in case state is skipped - if cancel, exists := c.cancelBy[txInfo.id]; exists { + if cancel, exists := c.cancelBy[info.id]; exists { cancel() // cancel context - delete(c.cancelBy, txInfo.id) + delete(c.cancelBy, info.id) } // delete from broadcasted map, if exists - delete(c.broadcastedProcessedTxs, txInfo.id) + delete(c.broadcastedProcessedTxs, info.id) // delete from confirmed map, if exists - delete(c.confirmedTxs, txInfo.id) + delete(c.confirmedTxs, info.id) // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic for _, s := range tx.signatures { delete(c.sigToTxInfo, s) @@ -399,15 +405,15 @@ func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout ti // if retention duration is set to 0, delete transaction from storage // otherwise, move to finalized map if retentionTimeout == 0 { - return txInfo.id, nil + return info.id, nil } finalizedTx := finishedTx{ state: Finalized, retentionTs: time.Now().Add(retentionTimeout), } // move transaction from confirmed to finalized map - c.finalizedErroredTxs[txInfo.id] = finalizedTx - return txInfo.id, nil + c.finalizedErroredTxs[info.id] = finalizedTx + return info.id, nil }) } @@ -455,14 +461,14 @@ func (c *pendingTxContext) OnPrebroadcastError(id string, retentionTimeout time. func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.Duration, txState TxState, _ TxErrType) (string, error) { err := c.withReadLock(func() error { - txInfo, sigExists := c.sigToTxInfo[sig] + info, sigExists := c.sigToTxInfo[sig] if !sigExists { return ErrSigDoesNotExist } // transaction can transition from any non-finalized state var broadcastedExists, confirmedExists bool - _, broadcastedExists = c.broadcastedProcessedTxs[txInfo.id] - _, confirmedExists = c.confirmedTxs[txInfo.id] + _, broadcastedExists = c.broadcastedProcessedTxs[info.id] + _, confirmedExists = c.confirmedTxs[info.id] // transcation does not exist in any tx maps if !broadcastedExists && !confirmedExists { return ErrTransactionNotFound @@ -475,16 +481,16 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D // upgrade to write lock if sig exists return c.withWriteLock(func() (string, error) { - txInfo, exists := c.sigToTxInfo[sig] + info, exists := c.sigToTxInfo[sig] if !exists { return "", ErrSigDoesNotExist } var tx, tempTx pendingTx var broadcastedExists, confirmedExists bool - if tempTx, broadcastedExists = c.broadcastedProcessedTxs[txInfo.id]; broadcastedExists { + if tempTx, broadcastedExists = c.broadcastedProcessedTxs[info.id]; broadcastedExists { tx = tempTx } - if tempTx, confirmedExists = c.confirmedTxs[txInfo.id]; confirmedExists { + if tempTx, confirmedExists = c.confirmedTxs[info.id]; confirmedExists { tx = tempTx } // transcation does not exist in any non-finalized maps @@ -492,29 +498,29 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D return "", ErrTransactionNotFound } // call cancel func + remove from map - if cancel, exists := c.cancelBy[txInfo.id]; exists { + if cancel, exists := c.cancelBy[info.id]; exists { cancel() // cancel context - delete(c.cancelBy, txInfo.id) + delete(c.cancelBy, info.id) } // delete from broadcasted map, if exists - delete(c.broadcastedProcessedTxs, txInfo.id) + delete(c.broadcastedProcessedTxs, info.id) // delete from confirmed map, if exists - delete(c.confirmedTxs, txInfo.id) + delete(c.confirmedTxs, info.id) // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic for _, s := range tx.signatures { delete(c.sigToTxInfo, s) } // if retention duration is set to 0, skip adding transaction to the errored map if retentionTimeout == 0 { - return txInfo.id, nil + return info.id, nil } erroredTx := finishedTx{ state: txState, retentionTs: time.Now().Add(retentionTimeout), } // move transaction from broadcasted to error map - c.finalizedErroredTxs[txInfo.id] = erroredTx - return txInfo.id, nil + c.finalizedErroredTxs[info.id] = erroredTx + return info.id, nil }) } @@ -561,6 +567,107 @@ func (c *pendingTxContext) TrimFinalizedErroredTxs() int { return len(expiredIDs) } +func (c *pendingTxContext) GetSignatureInfo(sig solana.Signature) (txInfo, error) { + c.lock.RLock() + defer c.lock.RUnlock() + + info, exists := c.sigToTxInfo[sig] + if !exists { + return txInfo{}, ErrSigDoesNotExist + } + return info, nil +} + +func (c *pendingTxContext) UpdateSignatureStatus(sig solana.Signature, newStatus TxState) (string, error) { + // First, acquire a read lock to check if the signature exists and needs to be updated + err := c.withReadLock(func() error { + info, exists := c.sigToTxInfo[sig] + if !exists { + return ErrSigDoesNotExist + } + if info.status == newStatus { + return ErrAlreadyInExpectedState + } + return nil + }) + if err != nil { + return "", err + } + + // Upgrade to a write lock to perform the update + return c.withWriteLock(func() (string, error) { + info, exists := c.sigToTxInfo[sig] + if !exists { + return "", ErrSigDoesNotExist + } + if info.status == newStatus { + // Another goroutine might have updated the status; no action needed + return "", ErrAlreadyInExpectedState + } + info.status = newStatus + c.sigToTxInfo[sig] = info + return "", nil + }) +} + +func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { + // Acquire a read lock to check if the signature exists and needs to be reset + err := c.withReadLock(func() error { + // Check if the signature is still being tracked + info, exists := c.sigToTxInfo[sig] + if !exists { + return ErrSigDoesNotExist + } + + // Check if the transaction is still in a non finalized/errored state + var broadcastedExists, confirmedExists bool + _, broadcastedExists = c.broadcastedProcessedTxs[info.id] + _, confirmedExists = c.confirmedTxs[info.id] + if !broadcastedExists && !confirmedExists { + return ErrTransactionNotFound + } + return nil + }) + if err != nil { + // If transaction or sig are not found, return + return pendingTx{}, err + } + + var pTx pendingTx + // Acquire a write lock to perform the state reset + _, err = c.withWriteLock(func() (string, error) { + // Retrieve sig and tx again inside the write lock + info, exists := c.sigToTxInfo[sig] + if !exists { + return "", ErrSigDoesNotExist + } + var tx pendingTx + var broadcastedExists, confirmedExists bool + if tx, broadcastedExists = c.broadcastedProcessedTxs[info.id]; broadcastedExists { + pTx = tx + } + if tx, confirmedExists = c.confirmedTxs[info.id]; confirmedExists { + pTx = tx + } + if !broadcastedExists && !confirmedExists { + // transcation does not exist in any non finalized/errored maps + return "", ErrTransactionNotFound + } + + // Reset the signature and tx status for retrying + info.status, pTx.state = Broadcasted, Broadcasted + c.sigToTxInfo[sig] = info + return "", nil + }) + if err != nil { + // If transaction or sig were not found, return + return pendingTx{}, err + } + + // Return the transaction for retrying + return pTx, nil +} + func (c *pendingTxContext) withReadLock(fn func() error) error { c.lock.RLock() defer c.lock.RUnlock() @@ -687,3 +794,15 @@ func (c *pendingTxContextWithProm) GetTxState(id string) (TxState, error) { func (c *pendingTxContextWithProm) TrimFinalizedErroredTxs() int { return c.pendingTx.TrimFinalizedErroredTxs() } + +func (c *pendingTxContextWithProm) GetSignatureInfo(sig solana.Signature) (txInfo, error) { + return c.pendingTx.GetSignatureInfo(sig) +} + +func (c *pendingTxContextWithProm) UpdateSignatureStatus(sig solana.Signature, newStatus TxState) (string, error) { + return c.pendingTx.UpdateSignatureStatus(sig, newStatus) +} + +func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, error) { + return c.pendingTx.OnReorg(sig) +} diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 3ba39f2f5..062f2ad40 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -224,18 +224,11 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran txm.lggr.Debugw("tx initial broadcast", "id", msg.id, "fee", msg.cfg.BaseComputeUnitPrice, "signature", sig) - // Initialize signature list with initialTx signature. This list will be used to add new signatures and track retry attempts. - sigs := &signatureList{} - sigs.Allocate() - if initSetErr := sigs.Set(0, sig); initSetErr != nil { - return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to save initial signature in signature list: %w", initSetErr) - } - // pass in copy of msg (to build new tx with bumped fee) and broadcasted tx == initTx (to retry tx without bumping) txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(ctx, msg, initTx, sigs) + txm.retryTx(ctx, msg, initTx, sig) }() // Return signed tx, id, signature for use in simulation @@ -286,7 +279,15 @@ func (txm *Txm) buildTx(ctx context.Context, msg pendingTx, retryCount int) (sol // retryTx contains the logic for retrying the transaction, including exponential backoff and fee bumping. // Retries until context cancelled by timeout or called externally. // It uses handleRetry helper function to handle each retry attempt. -func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sigs *signatureList) { +func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature) { + // Initialize signature list with initialTx signature. This list will be used to add new signatures and track retry attempts. + sigs := &signatureList{} + sigs.Allocate() + if initSetErr := sigs.Set(0, sig); initSetErr != nil { + txm.lggr.Errorw("failed to save initial signature in signature list", "error", initSetErr) + return + } + deltaT := 1 // initial delay in ms tick := time.After(0) bumpCount := 0 @@ -463,6 +464,12 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr continue } + // check if a potential re-org has occurred for this sig and handle it + err := txm.handleReorg(ctx, sig, status) + if err != nil { + continue + } + switch status.ConfirmationStatus { case rpc.ConfirmationStatusProcessed: // if signature is processed, keep polling for confirmed or finalized status @@ -522,6 +529,41 @@ func (txm *Txm) handleErrorSignatureStatus(sig solanaGo.Signature, status *rpc.S } } +// handleReorg handles the case where a transaction signature is in a potential reorg state on-chain. +// It updates the transaction state in the local memory and restarts the retry/bumping cycle for the transaction associated to that sig. +func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status *rpc.SignatureStatusesResult) error { + // Retrieve last seen status for the tx associated to this sig in our in-memory layer. + txInfo, err := txm.txs.GetSignatureInfo(sig) + if err != nil { + txm.lggr.Errorw("failed to get signature info when checking for potential re-orgs", "signature", sig, "error", err) + return err + } + + // Check if tx has been reorged by detecting if we had a status regression + // If so, we'll handle the reorg by updating the status in our in-memory layer and retrying the transaction for that sig. + currentTxState := convertStatus(status) + if isStatusRegression(txInfo.status, currentTxState) { + txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.status, "currentStatus", currentTxState) + // Update status for the tx associated to this sig in our in-memory layer with last seen on-chain status. + _, err = txm.txs.UpdateSignatureStatus(sig, currentTxState) + if err != nil { + txm.lggr.Errorw("failed to update signature status", "signature", sig, "error", err) + return err + } + + // Handle reorg in our in memory layer and retry transaction + pTx, err := txm.txs.OnReorg(sig) + if err != nil { + txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) + return err + } + retryCtx, _ := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: Ask here. How should we handle the ctx? + txm.retryTx(retryCtx, pTx, pTx.tx, sig) + } + + return nil +} + // handleProcessedSignatureStatus handles the case where a transaction signature is in the "processed" state on-chain. // It updates the transaction state in the local memory and checks if the confirmation timeout has been exceeded. // If the timeout is exceeded, it marks the transaction as errored. diff --git a/pkg/solana/txm/utils.go b/pkg/solana/txm/utils.go index fef260e3d..33e59a64e 100644 --- a/pkg/solana/txm/utils.go +++ b/pkg/solana/txm/utils.go @@ -111,6 +111,34 @@ func convertStatus(res *rpc.SignatureStatusesResult) TxState { return NotFound } +// isStatusRegression checks if the current status is a regression compared to the previous status: +// - Finalized -> Confirmed, Processed, Broadcasted: should not regress +// - Confirmed -> Processed, Broadcasted: should not regress +// - Processed -> Broadcasted: should not regress +// Returns true if a regression is detected, indicating a possible re-org. +func isStatusRegression(previous, current TxState) bool { + switch previous { + case Finalized: + // Finalized transactions should not regress. + if current != Finalized { + return true + } + case Confirmed: + // Confirmed transactions should not regress to Processed or Broadcasted. + if current != Confirmed && current != Finalized { + return true + } + case Processed: + // Processed transactions should not regress to Broadcasted. + if current != Processed && current != Confirmed && current != Finalized { + return true + } + default: + return false + } + return false +} + type signatureList struct { sigs []solana.Signature lock sync.RWMutex From 4fd327a257fd1d41dd383ec2f9121ba40463b2e4 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Fri, 29 Nov 2024 13:13:23 -0300 Subject: [PATCH 03/31] linting errors --- pkg/solana/txm/txm_internal_test.go | 65 +++++++++++++---------------- pkg/solana/txm/txm_load_test.go | 4 +- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 95017a29e..4f268aed2 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1329,24 +1329,22 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { // First transaction should be rebroadcasted. if time.Since(nowTs) < cfg.TxConfirmTimeout()-2*time.Second { return nil - } else { - // Second transaction should reach finalization. - sigStatusCallCount++ - if sigStatusCallCount == 1 { - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusProcessed, - } - } else if sigStatusCallCount == 2 { - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusConfirmed, - } - } else { - wg.Done() - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusFinalized, - } + } + // Second transaction should reach finalization. + sigStatusCallCount++ + if sigStatusCallCount == 1 { + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } else if sigStatusCallCount == 2 { + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, } } + wg.Done() + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusFinalized, + } } txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, slotHeightFunc, sendTxFunc, statuses) @@ -1396,10 +1394,9 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { // Transaction remains unconfirmed and should not be rebroadcasted. if time.Since(nowTs) < cfg.TxConfirmTimeout() { return nil - } else { - wg.Done() - return nil } + wg.Done() + return nil } txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, nil, sendTxFunc, statuses) @@ -1461,24 +1458,22 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { // transaction should be rebroadcasted multiple times. if time.Since(nowTs) < cfg.TxConfirmTimeout()-2*time.Second { return nil - } else { - // Second transaction should reach finalization. - sigStatusCallCount++ - if sigStatusCallCount == 1 { - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusProcessed, - } - } else if sigStatusCallCount == 2 { - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusConfirmed, - } - } else { - wg.Done() - return &rpc.SignatureStatusesResult{ - ConfirmationStatus: rpc.ConfirmationStatusFinalized, - } + } + // Second transaction should reach finalization. + sigStatusCallCount++ + if sigStatusCallCount == 1 { + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } else if sigStatusCallCount == 2 { + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, } } + wg.Done() + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusFinalized, + } } txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, slotHeightFunc, sendTxFunc, statuses) diff --git a/pkg/solana/txm/txm_load_test.go b/pkg/solana/txm/txm_load_test.go index aa3d6aac7..bc30ef92b 100644 --- a/pkg/solana/txm/txm_load_test.go +++ b/pkg/solana/txm/txm_load_test.go @@ -85,8 +85,8 @@ func TestTxm_Integration(t *testing.T) { assert.Error(t, txm.Start(ctx)) createTx := func(signer solana.PublicKey, sender solana.PublicKey, receiver solana.PublicKey, amt uint64) *solana.Transaction { // create transfer tx - hash, err := client.LatestBlockhash(ctx) - assert.NoError(t, err) + hash, hashErr := client.LatestBlockhash(ctx) + assert.NoError(t, hashErr) tx, txErr := solana.NewTransaction( []solana.Instruction{ system.NewTransferInstruction( From 2b29c3371c4446b4e326f9a865117f45f7633809 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Fri, 29 Nov 2024 14:59:06 -0300 Subject: [PATCH 04/31] fix some state tracking instances --- pkg/solana/txm/pendingtx.go | 32 +++++++++++++++++--------------- pkg/solana/txm/txm.go | 4 ++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 0f8d5bef1..7c950e9c6 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -73,8 +73,8 @@ type finishedTx struct { } type txInfo struct { - id string - status TxState + id string + state TxState } var _ PendingTxContext = &pendingTxContext{} @@ -127,7 +127,7 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex } // save cancel func c.cancelBy[tx.id] = cancel - c.sigToTxInfo[sig] = txInfo{id: tx.id, status: Broadcasted} + c.sigToTxInfo[sig] = txInfo{id: tx.id, state: Broadcasted} // add signature to tx tx.signatures = append(tx.signatures, sig) tx.createTs = time.Now() @@ -164,7 +164,7 @@ func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { if _, exists := c.broadcastedProcessedTxs[id]; !exists { return "", ErrTransactionNotFound } - c.sigToTxInfo[sig] = txInfo{id: id, status: Broadcasted} + c.sigToTxInfo[sig] = txInfo{id: id, state: Broadcasted} tx := c.broadcastedProcessedTxs[id] // save new signature tx.signatures = append(tx.signatures, sig) @@ -299,9 +299,10 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { if !exists { return info.id, ErrTransactionNotFound } - // update tx state to Processed - tx.state = Processed - // save updated tx back to the broadcasted map + // update sig and tx to Processed + info.state, tx.state = Processed, Processed + // save updated sig and tx back to the maps + c.sigToTxInfo[sig] = info c.broadcastedProcessedTxs[info.id] = tx return info.id, nil }) @@ -343,8 +344,9 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { cancel() // cancel context delete(c.cancelBy, info.id) } - // update tx state to Confirmed - tx.state = Confirmed + // update sig and tx state to Confirmed + info.state, tx.state = Confirmed, Confirmed + c.sigToTxInfo[sig] = info // move tx to confirmed map c.confirmedTxs[info.id] = tx // remove tx from broadcasted map @@ -585,7 +587,7 @@ func (c *pendingTxContext) UpdateSignatureStatus(sig solana.Signature, newStatus if !exists { return ErrSigDoesNotExist } - if info.status == newStatus { + if info.state == newStatus { return ErrAlreadyInExpectedState } return nil @@ -600,11 +602,11 @@ func (c *pendingTxContext) UpdateSignatureStatus(sig solana.Signature, newStatus if !exists { return "", ErrSigDoesNotExist } - if info.status == newStatus { - // Another goroutine might have updated the status; no action needed + if info.state == newStatus { + // no action needed return "", ErrAlreadyInExpectedState } - info.status = newStatus + info.state = newStatus c.sigToTxInfo[sig] = info return "", nil }) @@ -654,8 +656,8 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { return "", ErrTransactionNotFound } - // Reset the signature and tx status for retrying - info.status, pTx.state = Broadcasted, Broadcasted + // Reset the signature status and tx for retrying + info.state, pTx.state = Broadcasted, Broadcasted c.sigToTxInfo[sig] = info return "", nil }) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 062f2ad40..66bb3ddf6 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -542,8 +542,8 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status // Check if tx has been reorged by detecting if we had a status regression // If so, we'll handle the reorg by updating the status in our in-memory layer and retrying the transaction for that sig. currentTxState := convertStatus(status) - if isStatusRegression(txInfo.status, currentTxState) { - txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.status, "currentStatus", currentTxState) + if isStatusRegression(txInfo.state, currentTxState) { + txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) // Update status for the tx associated to this sig in our in-memory layer with last seen on-chain status. _, err = txm.txs.UpdateSignatureStatus(sig, currentTxState) if err != nil { From a6ce47b714c71b4f5ef28dcaf78753e1b64a1a4a Mon Sep 17 00:00:00 2001 From: Farber98 Date: Sat, 30 Nov 2024 13:54:25 -0300 Subject: [PATCH 05/31] remove redundant sig update --- pkg/solana/txm/pendingtx.go | 40 +------------------------------------ pkg/solana/txm/txm.go | 13 +++++------- 2 files changed, 6 insertions(+), 47 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 7c950e9c6..cba91c52f 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -49,9 +49,7 @@ type PendingTxContext interface { TrimFinalizedErroredTxs() int // GetSignatureInfo returns the transaction ID and TxState for the provided signature GetSignatureInfo(sig solana.Signature) (txInfo, error) - // UpdateSignatureStatus updates the status of the provided signature within sigToTxInfo map - UpdateSignatureStatus(sig solana.Signature, newStatus TxState) (string, error) - // OnReorg resets the transaction state to Broadcasted for the given signature and returns the pendingTx. + // OnReorg resets the transaction state to Broadcasted for the given signature and returns the pendingTx for retrying. OnReorg(sig solana.Signature) (pendingTx, error) } @@ -580,38 +578,6 @@ func (c *pendingTxContext) GetSignatureInfo(sig solana.Signature) (txInfo, error return info, nil } -func (c *pendingTxContext) UpdateSignatureStatus(sig solana.Signature, newStatus TxState) (string, error) { - // First, acquire a read lock to check if the signature exists and needs to be updated - err := c.withReadLock(func() error { - info, exists := c.sigToTxInfo[sig] - if !exists { - return ErrSigDoesNotExist - } - if info.state == newStatus { - return ErrAlreadyInExpectedState - } - return nil - }) - if err != nil { - return "", err - } - - // Upgrade to a write lock to perform the update - return c.withWriteLock(func() (string, error) { - info, exists := c.sigToTxInfo[sig] - if !exists { - return "", ErrSigDoesNotExist - } - if info.state == newStatus { - // no action needed - return "", ErrAlreadyInExpectedState - } - info.state = newStatus - c.sigToTxInfo[sig] = info - return "", nil - }) -} - func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { // Acquire a read lock to check if the signature exists and needs to be reset err := c.withReadLock(func() error { @@ -801,10 +767,6 @@ func (c *pendingTxContextWithProm) GetSignatureInfo(sig solana.Signature) (txInf return c.pendingTx.GetSignatureInfo(sig) } -func (c *pendingTxContextWithProm) UpdateSignatureStatus(sig solana.Signature, newStatus TxState) (string, error) { - return c.pendingTx.UpdateSignatureStatus(sig, newStatus) -} - func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, error) { return c.pendingTx.OnReorg(sig) } diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 66bb3ddf6..0da1576c9 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -544,13 +544,6 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status currentTxState := convertStatus(status) if isStatusRegression(txInfo.state, currentTxState) { txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) - // Update status for the tx associated to this sig in our in-memory layer with last seen on-chain status. - _, err = txm.txs.UpdateSignatureStatus(sig, currentTxState) - if err != nil { - txm.lggr.Errorw("failed to update signature status", "signature", sig, "error", err) - return err - } - // Handle reorg in our in memory layer and retry transaction pTx, err := txm.txs.OnReorg(sig) if err != nil { @@ -558,7 +551,11 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status return err } retryCtx, _ := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: Ask here. How should we handle the ctx? - txm.retryTx(retryCtx, pTx, pTx.tx, sig) + txm.done.Add(1) + go func() { + defer txm.done.Done() + txm.retryTx(retryCtx, pTx, pTx.tx, sig) + }() } return nil From 3a6e643c35b0a31332713099dded6b811226f692 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Sat, 30 Nov 2024 15:32:18 -0300 Subject: [PATCH 06/31] move state from txes to sigs --- pkg/solana/txm/pendingtx.go | 80 +++++++++++++++++++++-------- pkg/solana/txm/pendingtx_test.go | 6 +-- pkg/solana/txm/txm.go | 9 ++-- pkg/solana/txm/txm_internal_test.go | 5 +- 4 files changed, 68 insertions(+), 32 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index cba91c52f..91a468d67 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -60,7 +60,6 @@ type pendingTx struct { signatures []solana.Signature id string createTs time.Time - state TxState lastValidBlockHeight uint64 // to track expiration } @@ -129,7 +128,6 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex // add signature to tx tx.signatures = append(tx.signatures, sig) tx.createTs = time.Now() - tx.state = Broadcasted // save to the broadcasted map since transaction was just broadcasted c.broadcastedProcessedTxs[tx.id] = tx return "", nil @@ -237,7 +235,7 @@ func (c *pendingTxContext) ListAllExpiredBroadcastedTxs(currHeight uint64) []pen defer c.lock.RUnlock() broadcastedTxes := make([]pendingTx, 0, len(c.broadcastedProcessedTxs)) // worst case, all of them for _, tx := range c.broadcastedProcessedTxs { - if tx.state == Broadcasted && tx.lastValidBlockHeight < currHeight { + if tx.lastValidBlockHeight < currHeight { broadcastedTxes = append(broadcastedTxes, tx) } } @@ -273,12 +271,12 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { return ErrSigDoesNotExist } // Transactions should only move to processed from broadcasted - tx, exists := c.broadcastedProcessedTxs[info.id] + _, exists := c.broadcastedProcessedTxs[info.id] if !exists { return ErrTransactionNotFound } - // Check if tranasction already in processed state - if tx.state == Processed { + // Check if sig already in processed state + if info.state == Processed { return ErrAlreadyInExpectedState } return nil @@ -297,8 +295,8 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { if !exists { return info.id, ErrTransactionNotFound } - // update sig and tx to Processed - info.state, tx.state = Processed, Processed + // update sig to Processed + info.state = Processed // save updated sig and tx back to the maps c.sigToTxInfo[sig] = info c.broadcastedProcessedTxs[info.id] = tx @@ -313,8 +311,8 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { if !sigExists { return ErrSigDoesNotExist } - // Check if transaction already in confirmed state - if tx, exists := c.confirmedTxs[info.id]; exists && tx.state == Confirmed { + // Check if sig already in confirmed state + if _, exists := c.confirmedTxs[info.id]; exists && info.state == Confirmed { return ErrAlreadyInExpectedState } // Transactions should only move to confirmed from broadcasted/processed @@ -343,7 +341,7 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { delete(c.cancelBy, info.id) } // update sig and tx state to Confirmed - info.state, tx.state = Confirmed, Confirmed + info.state = Confirmed c.sigToTxInfo[sig] = info // move tx to confirmed map c.confirmedTxs[info.id] = tx @@ -524,21 +522,58 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D }) } +// GetTxState retrieves the aggregated state of a transaction based on all its signatures. +// It performs state aggregation only for transactions in broadcastedProcessedTxs or confirmedTxs. +// For transactions in finalizedErroredTxs, it directly returns the stored state. func (c *pendingTxContext) GetTxState(id string) (TxState, error) { c.lock.RLock() defer c.lock.RUnlock() + + // Check if the transaction exists in broadcastedProcessedTxs if tx, exists := c.broadcastedProcessedTxs[id]; exists { - return tx.state, nil + return c.aggregateTxState(tx), nil } + + // Check if the transaction exists in confirmedTxs if tx, exists := c.confirmedTxs[id]; exists { - return tx.state, nil + return c.aggregateTxState(tx), nil } + + // Check if the transaction exists in finalizedErroredTxs if tx, exists := c.finalizedErroredTxs[id]; exists { return tx.state, nil } + + // Transaction not found in any map return NotFound, fmt.Errorf("failed to find transaction for id: %s", id) } +// aggregateTxState determines the highest TxState among all signatures of a pending transaction. +func (c *pendingTxContext) aggregateTxState(tx pendingTx) TxState { + // Define the priority of states + statePriority := map[TxState]int{ + Broadcasted: 1, + Processed: 2, + Confirmed: 3, + } + + // Update highestState based on individual signature states + highestState := Broadcasted + for _, sig := range tx.signatures { + info, exists := c.sigToTxInfo[sig] + if !exists { + continue + } + if priority, ok := statePriority[info.state]; ok { + if priority > statePriority[highestState] { + highestState = info.state + } + } + } + + return highestState +} + // TrimFinalizedErroredTxs deletes transactions from the finalized/errored map and the allTxs map after the retention period has passed func (c *pendingTxContext) TrimFinalizedErroredTxs() int { var expiredIDs []string @@ -587,12 +622,11 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { return ErrSigDoesNotExist } - // Check if the transaction is still in a non finalized/errored state - var broadcastedExists, confirmedExists bool - _, broadcastedExists = c.broadcastedProcessedTxs[info.id] - _, confirmedExists = c.confirmedTxs[info.id] - if !broadcastedExists && !confirmedExists { - return ErrTransactionNotFound + // Check if the transaction is still in a non-finalized/non-errored state + if _, exists := c.broadcastedProcessedTxs[info.id]; !exists { + if _, exists := c.confirmedTxs[info.id]; !exists { + return ErrTransactionNotFound + } } return nil }) @@ -609,6 +643,8 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { if !exists { return "", ErrSigDoesNotExist } + + // Attempt to find the transaction in the broadcasted or confirmed maps var tx pendingTx var broadcastedExists, confirmedExists bool if tx, broadcastedExists = c.broadcastedProcessedTxs[info.id]; broadcastedExists { @@ -618,12 +654,12 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { pTx = tx } if !broadcastedExists && !confirmedExists { - // transcation does not exist in any non finalized/errored maps + // transaction does not exist in any non finalized/errored maps return "", ErrTransactionNotFound } - // Reset the signature status and tx for retrying - info.state, pTx.state = Broadcasted, Broadcasted + // Reset the signature status for retrying + info.state = Broadcasted c.sigToTxInfo[sig] = info return "", nil }) diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 60b208412..5baaf23a0 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -90,7 +90,7 @@ func TestPendingTxContext_new(t *testing.T) { require.Equal(t, sig, tx.signatures[0]) // Check status is Broadcasted - require.Equal(t, Broadcasted, tx.state) + require.Equal(t, Broadcasted, txInfo.state) // Check it does not exist in confirmed map _, exists = txs.confirmedTxs[msg.id] @@ -222,7 +222,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { require.Equal(t, sig, tx.signatures[0]) // Check status is Processed - require.Equal(t, Processed, tx.state) + require.Equal(t, Processed, txInfo.state) // Check it does not exist in confirmed map _, exists = txs.confirmedTxs[msg.id] @@ -361,7 +361,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { require.Equal(t, sig, tx.signatures[0]) // Check status is Confirmed - require.Equal(t, Confirmed, tx.state) + require.Equal(t, Confirmed, txInfo.state) // Check it does not exist in finalized map _, exists = txs.finalizedErroredTxs[msg.id] diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 0da1576c9..4c2e1cd6b 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -228,7 +228,7 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(ctx, msg, initTx, sig) + txm.retryTx(ctx, msg, initTx, sig, func() {}) }() // Return signed tx, id, signature for use in simulation @@ -279,7 +279,8 @@ func (txm *Txm) buildTx(ctx context.Context, msg pendingTx, retryCount int) (sol // retryTx contains the logic for retrying the transaction, including exponential backoff and fee bumping. // Retries until context cancelled by timeout or called externally. // It uses handleRetry helper function to handle each retry attempt. -func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature) { +func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature, cancel context.CancelFunc) { + defer cancel() // Initialize signature list with initialTx signature. This list will be used to add new signatures and track retry attempts. sigs := &signatureList{} sigs.Allocate() @@ -550,11 +551,11 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) return err } - retryCtx, _ := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: Ask here. How should we handle the ctx? + retryCtx, cancel := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: How should we handle the ctx? txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(retryCtx, pTx, pTx.tx, sig) + txm.retryTx(retryCtx, pTx, pTx.tx, sig, cancel) }() } diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 4f268aed2..b835d59f0 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1503,8 +1503,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { return sig1, nil } - // Mock LatestBlockhash to return an invalid blockhash less than slotHeight - // We won't use it as there will be no rebroadcasts txes to process. All txes will be confirmed before. + // There will be no rebroadcasts txes to process slotHeightFunc := func() (uint64, error) { return uint64(1500), nil } @@ -1514,7 +1513,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { defer func() { callCount++ }() return &rpc.GetLatestBlockhashResult{ Value: &rpc.LatestBlockhashResult{ - LastValidBlockHeight: uint64(1000), + LastValidBlockHeight: uint64(2000), }, }, nil } From f4c6069a6818bba90c73aed28b07909aac4859ea Mon Sep 17 00:00:00 2001 From: Farber98 Date: Sat, 30 Nov 2024 16:30:00 -0300 Subject: [PATCH 07/31] fix listAllExpiredBroadcastedTxs --- pkg/solana/txm/pendingtx.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 91a468d67..72a2e461f 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -233,13 +233,22 @@ func (c *pendingTxContext) ListAll() []solana.Signature { func (c *pendingTxContext) ListAllExpiredBroadcastedTxs(currHeight uint64) []pendingTx { c.lock.RLock() defer c.lock.RUnlock() - broadcastedTxes := make([]pendingTx, 0, len(c.broadcastedProcessedTxs)) // worst case, all of them - for _, tx := range c.broadcastedProcessedTxs { - if tx.lastValidBlockHeight < currHeight { - broadcastedTxes = append(broadcastedTxes, tx) + + expiredTxes := make([]pendingTx, 0, len(c.broadcastedProcessedTxs)) // worst case, all of them + + for id, tx := range c.broadcastedProcessedTxs { + state, err := c.GetTxState(id) + if err != nil { + continue // Ignore transactions that are not found + } + + // Check if the transaction is still in the Broadcasted state + if state == Broadcasted && tx.lastValidBlockHeight < currHeight { + expiredTxes = append(expiredTxes, tx) } } - return broadcastedTxes + + return expiredTxes } // Expired returns if the timeout for trying to confirm a signature has been reached From f027aebd096303382ad02be5b2565af24f2f80fc Mon Sep 17 00:00:00 2001 From: Farber98 Date: Sat, 30 Nov 2024 16:42:57 -0300 Subject: [PATCH 08/31] handle reorg after confirm cycle --- pkg/solana/txm/pendingtx.go | 3 +-- pkg/solana/txm/txm.go | 14 +++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 72a2e461f..f8b76830f 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -300,7 +300,7 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { if !sigExists { return info.id, ErrSigDoesNotExist } - tx, exists := c.broadcastedProcessedTxs[info.id] + _, exists := c.broadcastedProcessedTxs[info.id] if !exists { return info.id, ErrTransactionNotFound } @@ -308,7 +308,6 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { info.state = Processed // save updated sig and tx back to the maps c.sigToTxInfo[sig] = info - c.broadcastedProcessedTxs[info.id] = tx return info.id, nil }) } diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 4c2e1cd6b..d46d8c38b 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -465,27 +465,23 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr continue } - // check if a potential re-org has occurred for this sig and handle it - err := txm.handleReorg(ctx, sig, status) - if err != nil { - continue - } - switch status.ConfirmationStatus { case rpc.ConfirmationStatusProcessed: // if signature is processed, keep polling for confirmed or finalized status txm.handleProcessedSignatureStatus(sig) - continue case rpc.ConfirmationStatusConfirmed: // if signature is confirmed, keep polling for finalized status txm.handleConfirmedSignatureStatus(sig) - continue case rpc.ConfirmationStatusFinalized: // if signature is finalized, end polling txm.handleFinalizedSignatureStatus(sig) - continue default: txm.lggr.Warnw("unknown confirmation status", "signature", sig, "status", status.ConfirmationStatus) + } + + // check if a potential re-org has occurred for this sig and handle it + err := txm.handleReorg(ctx, sig, status) + if err != nil { continue } } From 8c18891f3d21b922733bcede809fc216f1aba58a Mon Sep 17 00:00:00 2001 From: Farber98 Date: Sun, 1 Dec 2024 23:24:52 -0300 Subject: [PATCH 09/31] associate sigs to retry ctx --- pkg/solana/txm/pendingtx.go | 110 +++++++++-------- pkg/solana/txm/pendingtx_test.go | 199 +++++++++++++++++++++---------- pkg/solana/txm/txm.go | 32 +++-- 3 files changed, 214 insertions(+), 127 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index f8b76830f..d4c9b2120 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -21,9 +21,10 @@ var ( type PendingTxContext interface { // New adds a new tranasction in Broadcasted state to the storage - New(msg pendingTx, sig solana.Signature, cancel context.CancelFunc) error - // AddSignature adds a new signature for an existing transaction ID - AddSignature(id string, sig solana.Signature) error + New(msg pendingTx) error + // AddSignature adds a new signature to a broadcasted transaction in the pending transaction context. + // It associates the provided context and cancel function with the signature to manage retry and bumping cycles. + AddSignature(ctx context.Context, cancel context.CancelFunc, id string, sig solana.Signature) error // Remove removes transaction and related signatures from storage if not in finalized or errored state Remove(sig solana.Signature) (string, error) // ListAll returns all of the signatures being tracked for all transactions not yet finalized or errored @@ -50,7 +51,7 @@ type PendingTxContext interface { // GetSignatureInfo returns the transaction ID and TxState for the provided signature GetSignatureInfo(sig solana.Signature) (txInfo, error) // OnReorg resets the transaction state to Broadcasted for the given signature and returns the pendingTx for retrying. - OnReorg(sig solana.Signature) (pendingTx, error) + OnReorg(sig solana.Signature) (pendingTx, retryCtx, error) } // finishedTx is used to store info required to track transactions to finality or error @@ -74,11 +75,16 @@ type txInfo struct { state TxState } +type retryCtx struct { + ctx context.Context + cancel context.CancelFunc +} + var _ PendingTxContext = &pendingTxContext{} type pendingTxContext struct { - cancelBy map[string]context.CancelFunc - sigToTxInfo map[solana.Signature]txInfo + sigToRetryCtx map[solana.Signature]retryCtx + sigToTxInfo map[solana.Signature]txInfo broadcastedProcessedTxs map[string]pendingTx // broadcasted and processed transactions that may require retry and bumping confirmedTxs map[string]pendingTx // transactions that require monitoring for re-org @@ -89,8 +95,8 @@ type pendingTxContext struct { func newPendingTxContext() *pendingTxContext { return &pendingTxContext{ - cancelBy: map[string]context.CancelFunc{}, - sigToTxInfo: map[solana.Signature]txInfo{}, + sigToRetryCtx: map[solana.Signature]retryCtx{}, + sigToTxInfo: map[solana.Signature]txInfo{}, broadcastedProcessedTxs: map[string]pendingTx{}, confirmedTxs: map[string]pendingTx{}, @@ -98,12 +104,9 @@ func newPendingTxContext() *pendingTxContext { } } -func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel context.CancelFunc) error { +// New adds a new tranasction in Broadcasted state to the storage +func (c *pendingTxContext) New(tx pendingTx) error { err := c.withReadLock(func() error { - // validate signature does not exist - if _, exists := c.sigToTxInfo[sig]; exists { - return ErrSigAlreadyExists - } // validate id does not exist if _, exists := c.broadcastedProcessedTxs[tx.id]; exists { return ErrIDAlreadyExists @@ -114,19 +117,12 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex return err } - // upgrade to write lock if sig or id do not exist + // upgrade to write lock if id do not exist _, err = c.withWriteLock(func() (string, error) { - if _, exists := c.sigToTxInfo[sig]; exists { - return "", ErrSigAlreadyExists - } if _, exists := c.broadcastedProcessedTxs[tx.id]; exists { return "", ErrIDAlreadyExists } - // save cancel func - c.cancelBy[tx.id] = cancel - c.sigToTxInfo[sig] = txInfo{id: tx.id, state: Broadcasted} - // add signature to tx - tx.signatures = append(tx.signatures, sig) + tx.signatures = []solana.Signature{} tx.createTs = time.Now() // save to the broadcasted map since transaction was just broadcasted c.broadcastedProcessedTxs[tx.id] = tx @@ -135,7 +131,9 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex return err } -func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { +// AddSignature adds a new signature to a broadcasted transaction in the pending transaction context. +// Additionally, it associates the provided context and cancel function with the signature to manage retry and bumping cycles. +func (c *pendingTxContext) AddSignature(ctx context.Context, cancel context.CancelFunc, id string, sig solana.Signature) error { err := c.withReadLock(func() error { // signature already exists if _, exists := c.sigToTxInfo[sig]; exists { @@ -164,6 +162,8 @@ func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { tx := c.broadcastedProcessedTxs[id] // save new signature tx.signatures = append(tx.signatures, sig) + // save retryCtx to stop retry/bumping cycles + c.sigToRetryCtx[sig] = retryCtx{ctx: ctx, cancel: cancel} // save updated tx to broadcasted map c.broadcastedProcessedTxs[id] = tx return "", nil @@ -208,15 +208,13 @@ func (c *pendingTxContext) Remove(sig solana.Signature) (id string, err error) { delete(c.confirmedTxs, info.id) } - // call cancel func + remove from map - if cancel, exists := c.cancelBy[info.id]; exists { - cancel() // cancel context - delete(c.cancelBy, info.id) - } - - // remove all signatures associated with transaction from sig map + // remove all signatures associated with transaction from sig map and cancel any associated contexts to stop retry/bumping cycles for _, s := range tx.signatures { delete(c.sigToTxInfo, s) + if rtryCtx, exists := c.sigToRetryCtx[s]; exists { + rtryCtx.cancel() + delete(c.sigToRetryCtx, s) + } } return info.id, nil }) @@ -344,9 +342,9 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { return info.id, ErrTransactionNotFound } // call cancel func + remove from map to stop the retry/bumping cycle for this transaction - if cancel, exists := c.cancelBy[info.id]; exists { - cancel() // cancel context - delete(c.cancelBy, info.id) + if rtryCtx, exists := c.sigToRetryCtx[sig]; exists { + rtryCtx.cancel() + delete(c.sigToRetryCtx, sig) } // update sig and tx state to Confirmed info.state = Confirmed @@ -394,19 +392,18 @@ func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout ti if !broadcastedExists && !confirmedExists { return info.id, ErrTransactionNotFound } - // call cancel func + remove from map to stop the retry/bumping cycle for this transaction - // cancel is expected to be called and removed when tx is confirmed but checked here too in case state is skipped - if cancel, exists := c.cancelBy[info.id]; exists { - cancel() // cancel context - delete(c.cancelBy, info.id) - } // delete from broadcasted map, if exists delete(c.broadcastedProcessedTxs, info.id) // delete from confirmed map, if exists delete(c.confirmedTxs, info.id) // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic + // call cancel func + remove from map to stop the retry/bumping cycle for this transaction for _, s := range tx.signatures { delete(c.sigToTxInfo, s) + if rtryCtx, exists := c.sigToRetryCtx[s]; exists { + rtryCtx.cancel() + delete(c.sigToRetryCtx, s) + } } // if retention duration is set to 0, delete transaction from storage // otherwise, move to finalized map @@ -503,18 +500,18 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D if !broadcastedExists && !confirmedExists { return "", ErrTransactionNotFound } - // call cancel func + remove from map - if cancel, exists := c.cancelBy[info.id]; exists { - cancel() // cancel context - delete(c.cancelBy, info.id) - } // delete from broadcasted map, if exists delete(c.broadcastedProcessedTxs, info.id) // delete from confirmed map, if exists delete(c.confirmedTxs, info.id) // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic + // call cancel func + remove from map to stop the retry/bumping cycle for this transaction for _, s := range tx.signatures { delete(c.sigToTxInfo, s) + if rtryCtx, exists := c.sigToRetryCtx[s]; exists { + rtryCtx.cancel() // cancel context + delete(c.sigToRetryCtx, s) + } } // if retention duration is set to 0, skip adding transaction to the errored map if retentionTimeout == 0 { @@ -621,7 +618,7 @@ func (c *pendingTxContext) GetSignatureInfo(sig solana.Signature) (txInfo, error return info, nil } -func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { +func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, retryCtx, error) { // Acquire a read lock to check if the signature exists and needs to be reset err := c.withReadLock(func() error { // Check if the signature is still being tracked @@ -640,10 +637,11 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { }) if err != nil { // If transaction or sig are not found, return - return pendingTx{}, err + return pendingTx{}, retryCtx{}, err } var pTx pendingTx + var rtryCtx retryCtx // Acquire a write lock to perform the state reset _, err = c.withWriteLock(func() (string, error) { // Retrieve sig and tx again inside the write lock @@ -652,6 +650,12 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { return "", ErrSigDoesNotExist } + // Check if the retryCtx is still relevant + rtryCtx, exists = c.sigToRetryCtx[sig] + if !exists { + return "", fmt.Errorf("retry context not found for signature %s associated to tx id %s", sig.String(), info.id) + } + // Attempt to find the transaction in the broadcasted or confirmed maps var tx pendingTx var broadcastedExists, confirmedExists bool @@ -673,11 +677,11 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { }) if err != nil { // If transaction or sig were not found, return - return pendingTx{}, err + return pendingTx{}, rtryCtx, err } // Return the transaction for retrying - return pTx, nil + return pTx, rtryCtx, nil } func (c *pendingTxContext) withReadLock(fn func() error) error { @@ -717,12 +721,12 @@ func newPendingTxContextWithProm(id string) *pendingTxContextWithProm { } } -func (c *pendingTxContextWithProm) New(msg pendingTx, sig solana.Signature, cancel context.CancelFunc) error { - return c.pendingTx.New(msg, sig, cancel) +func (c *pendingTxContextWithProm) New(msg pendingTx) error { + return c.pendingTx.New(msg) } -func (c *pendingTxContextWithProm) AddSignature(id string, sig solana.Signature) error { - return c.pendingTx.AddSignature(id, sig) +func (c *pendingTxContextWithProm) AddSignature(ctx context.Context, cancel context.CancelFunc, id string, sig solana.Signature) error { + return c.pendingTx.AddSignature(ctx, cancel, id, sig) } func (c *pendingTxContextWithProm) OnProcessed(sig solana.Signature) (string, error) { @@ -811,6 +815,6 @@ func (c *pendingTxContextWithProm) GetSignatureInfo(sig solana.Signature) (txInf return c.pendingTx.GetSignatureInfo(sig) } -func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, error) { +func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, retryCtx, error) { return c.pendingTx.OnReorg(sig) } diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 5baaf23a0..762630d52 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -40,13 +40,15 @@ func TestPendingTxContext_add_remove_multiple(t *testing.T) { for i := 0; i < n; i++ { sig, cancel := newProcess() msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + assert.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) assert.NoError(t, err) ids[sig] = msg.id } // cannot add signature for non existent ID - require.Error(t, txs.AddSignature(uuid.New().String(), solana.Signature{})) + require.Error(t, txs.AddSignature(ctx, func() {}, uuid.New().String(), solana.Signature{})) // return list of signatures list := txs.ListAll() @@ -69,13 +71,15 @@ func TestPendingTxContext_add_remove_multiple(t *testing.T) { func TestPendingTxContext_new(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) sig := randomSignature(t) txs := newPendingTxContext() // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Check it exists in signature map @@ -103,7 +107,7 @@ func TestPendingTxContext_new(t *testing.T) { func TestPendingTxContext_add_signature(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() t.Run("successfully add signature to transaction", func(t *testing.T) { @@ -112,10 +116,12 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig1) require.NoError(t, err) - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(ctx, cancel, msg.id, sig2) require.NoError(t, err) // Check signature map @@ -147,10 +153,12 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) - err = txs.AddSignature(msg.id, sig) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.ErrorIs(t, err, ErrSigAlreadyExists) }) @@ -160,10 +168,12 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig1) require.NoError(t, err) - err = txs.AddSignature("bad id", sig2) + err = txs.AddSignature(ctx, cancel, "bad id", sig2) require.ErrorIs(t, err, ErrTransactionNotFound) }) @@ -173,7 +183,9 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig1) require.NoError(t, err) // Transition to processed state @@ -186,14 +198,14 @@ func TestPendingTxContext_add_signature(t *testing.T) { require.NoError(t, err) require.Equal(t, msg.id, id) - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(ctx, cancel, msg.id, sig2) require.ErrorIs(t, err, ErrTransactionNotFound) }) } func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -202,7 +214,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -238,7 +252,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -261,7 +277,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -289,7 +307,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -307,7 +327,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -323,7 +345,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { func TestPendingTxContext_on_confirmed(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -332,7 +354,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -373,7 +397,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -401,7 +427,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -419,7 +447,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -440,7 +470,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { func TestPendingTxContext_on_finalized(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -450,11 +480,13 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig1) require.NoError(t, err) // Add second signature - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(ctx, cancel, msg.id, sig2) require.NoError(t, err) // Transition to finalized state @@ -490,11 +522,13 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig1) require.NoError(t, err) // Add second signature - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(ctx, cancel, msg.id, sig2) require.NoError(t, err) // Transition to processed state @@ -539,7 +573,9 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig1) require.NoError(t, err) // Transition to processed state @@ -579,7 +615,9 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -595,7 +633,7 @@ func TestPendingTxContext_on_finalized(t *testing.T) { func TestPendingTxContext_on_error(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -604,7 +642,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -637,7 +677,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -675,7 +717,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to fatally errored state @@ -704,7 +748,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -739,7 +785,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to confirmed state @@ -756,7 +804,7 @@ func TestPendingTxContext_on_error(t *testing.T) { func TestPendingTxContext_on_prebroadcast_error(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -797,7 +845,9 @@ func TestPendingTxContext_on_prebroadcast_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} // Add transaction to broadcasted map - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -820,7 +870,7 @@ func TestPendingTxContext_on_prebroadcast_error(t *testing.T) { func TestPendingTxContext_remove(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -834,14 +884,18 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new broadcasted transaction with extra sig broadcastedMsg := pendingTx{id: uuid.NewString()} - err := txs.New(broadcastedMsg, broadcastedSig1, cancel) + err := txs.New(broadcastedMsg) require.NoError(t, err) - err = txs.AddSignature(broadcastedMsg.id, broadcastedSig2) + err = txs.AddSignature(ctx, cancel, broadcastedMsg.id, broadcastedSig1) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, broadcastedMsg.id, broadcastedSig2) require.NoError(t, err) // Create new processed transaction processedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(processedMsg, processedSig, cancel) + err = txs.New(processedMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, processedMsg.id, processedSig) require.NoError(t, err) id, err := txs.OnProcessed(processedSig) require.NoError(t, err) @@ -849,7 +903,9 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new confirmed transaction confirmedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(confirmedMsg, confirmedSig, cancel) + err = txs.New(confirmedMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, confirmedMsg.id, confirmedSig) require.NoError(t, err) id, err = txs.OnConfirmed(confirmedSig) require.NoError(t, err) @@ -857,7 +913,9 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new finalized transaction finalizedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(finalizedMsg, finalizedSig, cancel) + err = txs.New(finalizedMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, finalizedMsg.id, finalizedSig) require.NoError(t, err) id, err = txs.OnFinalized(finalizedSig, retentionTimeout) require.NoError(t, err) @@ -865,7 +923,9 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new errored transaction erroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(erroredMsg, erroredSig, cancel) + err = txs.New(erroredMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, erroredMsg.id, erroredSig) require.NoError(t, err) id, err = txs.OnError(erroredSig, retentionTimeout, Errored, 0) require.NoError(t, err) @@ -919,6 +979,7 @@ func TestPendingTxContext_remove(t *testing.T) { // Check sig list is empty after all removals require.Empty(t, txs.ListAll()) } + func TestPendingTxContext_trim_finalized_errored_txs(t *testing.T) { t.Parallel() txs := newPendingTxContext() @@ -956,12 +1017,14 @@ func TestPendingTxContext_trim_finalized_errored_txs(t *testing.T) { func TestPendingTxContext_expired(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) sig := solana.Signature{} txs := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + assert.NoError(t, err) + err = txs.AddSignature(ctx, cancel, msg.id, sig) assert.NoError(t, err) msg, exists := txs.broadcastedProcessedTxs[msg.id] @@ -984,16 +1047,17 @@ func TestPendingTxContext_expired(t *testing.T) { func TestPendingTxContext_race(t *testing.T) { t.Run("new", func(t *testing.T) { txCtx := newPendingTxContext() + txID := uuid.NewString() var wg sync.WaitGroup wg.Add(2) var err [2]error go func() { - err[0] = txCtx.New(pendingTx{id: uuid.NewString()}, solana.Signature{}, func() {}) + err[0] = txCtx.New(pendingTx{id: txID}) wg.Done() }() go func() { - err[1] = txCtx.New(pendingTx{id: uuid.NewString()}, solana.Signature{}, func() {}) + err[1] = txCtx.New(pendingTx{id: txID}) wg.Done() }() @@ -1004,18 +1068,19 @@ func TestPendingTxContext_race(t *testing.T) { t.Run("add signature", func(t *testing.T) { txCtx := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - createErr := txCtx.New(msg, solana.Signature{}, func() {}) + createErr := txCtx.New(msg) require.NoError(t, createErr) + ctx, cancel := context.WithCancel(tests.Context(t)) var wg sync.WaitGroup wg.Add(2) var err [2]error go func() { - err[0] = txCtx.AddSignature(msg.id, solana.Signature{1}) + err[0] = txCtx.AddSignature(ctx, cancel, msg.id, solana.Signature{1}) wg.Done() }() go func() { - err[1] = txCtx.AddSignature(msg.id, solana.Signature{1}) + err[1] = txCtx.AddSignature(ctx, cancel, msg.id, solana.Signature{1}) wg.Done() }() @@ -1026,7 +1091,7 @@ func TestPendingTxContext_race(t *testing.T) { t.Run("remove", func(t *testing.T) { txCtx := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - err := txCtx.New(msg, solana.Signature{}, func() {}) + err := txCtx.New(msg) require.NoError(t, err) var wg sync.WaitGroup wg.Add(2) @@ -1046,7 +1111,7 @@ func TestPendingTxContext_race(t *testing.T) { func TestGetTxState(t *testing.T) { t.Parallel() - _, cancel := context.WithCancel(tests.Context(t)) + ctx, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -1059,13 +1124,17 @@ func TestGetTxState(t *testing.T) { // Create new broadcasted transaction with extra sig broadcastedMsg := pendingTx{id: uuid.NewString()} - err := txs.New(broadcastedMsg, broadcastedSig, cancel) + err := txs.New(broadcastedMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, broadcastedMsg.id, broadcastedSig) require.NoError(t, err) var state TxState // Create new processed transaction processedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(processedMsg, processedSig, cancel) + err = txs.New(processedMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, processedMsg.id, processedSig) require.NoError(t, err) id, err := txs.OnProcessed(processedSig) require.NoError(t, err) @@ -1077,7 +1146,9 @@ func TestGetTxState(t *testing.T) { // Create new confirmed transaction confirmedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(confirmedMsg, confirmedSig, cancel) + err = txs.New(confirmedMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, confirmedMsg.id, confirmedSig) require.NoError(t, err) id, err = txs.OnConfirmed(confirmedSig) require.NoError(t, err) @@ -1089,7 +1160,9 @@ func TestGetTxState(t *testing.T) { // Create new finalized transaction finalizedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(finalizedMsg, finalizedSig, cancel) + err = txs.New(finalizedMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, finalizedMsg.id, finalizedSig) require.NoError(t, err) id, err = txs.OnFinalized(finalizedSig, retentionTimeout) require.NoError(t, err) @@ -1101,7 +1174,9 @@ func TestGetTxState(t *testing.T) { // Create new errored transaction erroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(erroredMsg, erroredSig, cancel) + err = txs.New(erroredMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, erroredMsg.id, erroredSig) require.NoError(t, err) id, err = txs.OnError(erroredSig, retentionTimeout, Errored, 0) require.NoError(t, err) @@ -1113,7 +1188,9 @@ func TestGetTxState(t *testing.T) { // Create new fatally errored transaction fatallyErroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(fatallyErroredMsg, fatallyErroredSig, cancel) + err = txs.New(fatallyErroredMsg) + require.NoError(t, err) + err = txs.AddSignature(ctx, cancel, fatallyErroredMsg.id, fatallyErroredSig) require.NoError(t, err) id, err = txs.OnError(fatallyErroredSig, retentionTimeout, FatallyErrored, 0) require.NoError(t, err) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index d46d8c38b..77c354771 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -216,10 +216,16 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("tx failed initial transmit: %w", errors.Join(initSendErr, stateTransitionErr)) } - // Store tx signature and cancel function - if err := txm.txs.New(msg, sig, cancel); err != nil { - cancel() // Cancel context when exiting early - return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to save tx signature (%s) to inflight txs: %w", sig, err) + // Create new transaction in memory + if err := txm.txs.New(msg); err != nil { + cancel() + return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to create new transaction in memory: %w", err) + } + + // Associate signature and retryCtx to tx + if err := txm.txs.AddSignature(ctx, cancel, msg.id, sig); err != nil { + cancel() + return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to save initial signature (%s) to inflight txs: %w", sig, err) } txm.lggr.Debugw("tx initial broadcast", "id", msg.id, "fee", msg.cfg.BaseComputeUnitPrice, "signature", sig) @@ -228,7 +234,7 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(ctx, msg, initTx, sig, func() {}) + txm.retryTx(ctx, cancel, msg, initTx, sig) }() // Return signed tx, id, signature for use in simulation @@ -279,8 +285,7 @@ func (txm *Txm) buildTx(ctx context.Context, msg pendingTx, retryCount int) (sol // retryTx contains the logic for retrying the transaction, including exponential backoff and fee bumping. // Retries until context cancelled by timeout or called externally. // It uses handleRetry helper function to handle each retry attempt. -func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature, cancel context.CancelFunc) { - defer cancel() +func (txm *Txm) retryTx(ctx context.Context, cancel context.CancelFunc, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature) { // Initialize signature list with initialTx signature. This list will be used to add new signatures and track retry attempts. sigs := &signatureList{} sigs.Allocate() @@ -329,7 +334,7 @@ func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.T wg.Add(1) go func(bump bool, count int, retryTx solanaGo.Transaction) { defer wg.Done() - txm.handleRetry(ctx, msg, bump, count, retryTx, sigs) + txm.handleRetry(ctx, cancel, msg, bump, count, retryTx, sigs) }(shouldBump, bumpCount, currentTx) } @@ -343,7 +348,7 @@ func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.T } // handleRetry handles the logic for each retry attempt, including sending the transaction, updating signatures, and logging. -func (txm *Txm) handleRetry(ctx context.Context, msg pendingTx, bump bool, count int, retryTx solanaGo.Transaction, sigs *signatureList) { +func (txm *Txm) handleRetry(ctx context.Context, cancel context.CancelFunc, msg pendingTx, bump bool, count int, retryTx solanaGo.Transaction, sigs *signatureList) { // send retry transaction retrySig, err := txm.sendTx(ctx, &retryTx) if err != nil { @@ -358,7 +363,7 @@ func (txm *Txm) handleRetry(ctx context.Context, msg pendingTx, bump bool, count // if bump is true, update signature list and set new signature in space already allocated. if bump { - if err := txm.txs.AddSignature(msg.id, retrySig); err != nil { + if err := txm.txs.AddSignature(ctx, cancel, msg.id, retrySig); err != nil { txm.lggr.Warnw("error in adding retry transaction", "error", err, "id", msg.id) return } @@ -542,16 +547,17 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status if isStatusRegression(txInfo.state, currentTxState) { txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) // Handle reorg in our in memory layer and retry transaction - pTx, err := txm.txs.OnReorg(sig) + // We'll retrieve the associated pendgingTx and retryCtx to the sig + // attempting to restart the retry/bumping cycle for it if it's still in-flight + pTx, retryCtx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) return err } - retryCtx, cancel := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: How should we handle the ctx? txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(retryCtx, pTx, pTx.tx, sig, cancel) + txm.retryTx(retryCtx.ctx, retryCtx.cancel, pTx, pTx.tx, sig) }() } From 2902ec0b9d3450408d3117ce2aa16cbcfcc1195f Mon Sep 17 00:00:00 2001 From: Farber98 Date: Mon, 2 Dec 2024 00:02:22 -0300 Subject: [PATCH 10/31] remove unused ctx --- pkg/solana/txm/txm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 77c354771..b70e9973d 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -485,7 +485,7 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr } // check if a potential re-org has occurred for this sig and handle it - err := txm.handleReorg(ctx, sig, status) + err := txm.handleReorg(sig, status) if err != nil { continue } @@ -533,7 +533,7 @@ func (txm *Txm) handleErrorSignatureStatus(sig solanaGo.Signature, status *rpc.S // handleReorg handles the case where a transaction signature is in a potential reorg state on-chain. // It updates the transaction state in the local memory and restarts the retry/bumping cycle for the transaction associated to that sig. -func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status *rpc.SignatureStatusesResult) error { +func (txm *Txm) handleReorg(sig solanaGo.Signature, status *rpc.SignatureStatusesResult) error { // Retrieve last seen status for the tx associated to this sig in our in-memory layer. txInfo, err := txm.txs.GetSignatureInfo(sig) if err != nil { From 1c1f723bda42fedadc379c12ef45c361394479b5 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Mon, 2 Dec 2024 12:20:12 -0300 Subject: [PATCH 11/31] add errored state and remove finalized --- pkg/solana/txm/utils.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/solana/txm/utils.go b/pkg/solana/txm/utils.go index 33e59a64e..067e16bbd 100644 --- a/pkg/solana/txm/utils.go +++ b/pkg/solana/txm/utils.go @@ -112,25 +112,19 @@ func convertStatus(res *rpc.SignatureStatusesResult) TxState { } // isStatusRegression checks if the current status is a regression compared to the previous status: -// - Finalized -> Confirmed, Processed, Broadcasted: should not regress // - Confirmed -> Processed, Broadcasted: should not regress // - Processed -> Broadcasted: should not regress // Returns true if a regression is detected, indicating a possible re-org. func isStatusRegression(previous, current TxState) bool { switch previous { - case Finalized: - // Finalized transactions should not regress. - if current != Finalized { - return true - } case Confirmed: // Confirmed transactions should not regress to Processed or Broadcasted. - if current != Confirmed && current != Finalized { + if current != Confirmed && current != Finalized && current != Errored { return true } case Processed: // Processed transactions should not regress to Broadcasted. - if current != Processed && current != Confirmed && current != Finalized { + if current != Processed && current != Confirmed && current != Finalized && current != Errored { return true } default: From 6bc0c62da48d164e3e13e85edc7116d0f5025b4a Mon Sep 17 00:00:00 2001 From: Farber98 Date: Mon, 2 Dec 2024 12:33:31 -0300 Subject: [PATCH 12/31] comment --- pkg/solana/txm/txm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index b70e9973d..36b8af62a 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -548,7 +548,7 @@ func (txm *Txm) handleReorg(sig solanaGo.Signature, status *rpc.SignatureStatuse txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) // Handle reorg in our in memory layer and retry transaction // We'll retrieve the associated pendgingTx and retryCtx to the sig - // attempting to restart the retry/bumping cycle for it if it's still in-flight + // attempting to restart the retry/bumping cycle for it if retryCtx still valid pTx, retryCtx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) From 05442b267402aea5cbd891b1dbce0c73dced9cd3 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 13:24:42 -0300 Subject: [PATCH 13/31] Revert "comment" This reverts commit 6bc0c62da48d164e3e13e85edc7116d0f5025b4a. --- pkg/solana/txm/txm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 36b8af62a..b70e9973d 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -548,7 +548,7 @@ func (txm *Txm) handleReorg(sig solanaGo.Signature, status *rpc.SignatureStatuse txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) // Handle reorg in our in memory layer and retry transaction // We'll retrieve the associated pendgingTx and retryCtx to the sig - // attempting to restart the retry/bumping cycle for it if retryCtx still valid + // attempting to restart the retry/bumping cycle for it if it's still in-flight pTx, retryCtx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) From 9b27a5b0da2c04c830f69ac0267e2d2b61f09450 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 13:25:23 -0300 Subject: [PATCH 14/31] Revert "remove unused ctx" This reverts commit 2902ec0b9d3450408d3117ce2aa16cbcfcc1195f. --- pkg/solana/txm/txm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index b70e9973d..77c354771 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -485,7 +485,7 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr } // check if a potential re-org has occurred for this sig and handle it - err := txm.handleReorg(sig, status) + err := txm.handleReorg(ctx, sig, status) if err != nil { continue } @@ -533,7 +533,7 @@ func (txm *Txm) handleErrorSignatureStatus(sig solanaGo.Signature, status *rpc.S // handleReorg handles the case where a transaction signature is in a potential reorg state on-chain. // It updates the transaction state in the local memory and restarts the retry/bumping cycle for the transaction associated to that sig. -func (txm *Txm) handleReorg(sig solanaGo.Signature, status *rpc.SignatureStatusesResult) error { +func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status *rpc.SignatureStatusesResult) error { // Retrieve last seen status for the tx associated to this sig in our in-memory layer. txInfo, err := txm.txs.GetSignatureInfo(sig) if err != nil { From ee14b60e00815bfc60a4b523ed786dd8d9220275 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 13:28:37 -0300 Subject: [PATCH 15/31] Revert "associate sigs to retry ctx" This reverts commit 8c18891f3d21b922733bcede809fc216f1aba58a. --- pkg/solana/txm/pendingtx.go | 110 ++++++++--------- pkg/solana/txm/pendingtx_test.go | 199 ++++++++++--------------------- pkg/solana/txm/txm.go | 32 ++--- 3 files changed, 127 insertions(+), 214 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index d4c9b2120..f8b76830f 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -21,10 +21,9 @@ var ( type PendingTxContext interface { // New adds a new tranasction in Broadcasted state to the storage - New(msg pendingTx) error - // AddSignature adds a new signature to a broadcasted transaction in the pending transaction context. - // It associates the provided context and cancel function with the signature to manage retry and bumping cycles. - AddSignature(ctx context.Context, cancel context.CancelFunc, id string, sig solana.Signature) error + New(msg pendingTx, sig solana.Signature, cancel context.CancelFunc) error + // AddSignature adds a new signature for an existing transaction ID + AddSignature(id string, sig solana.Signature) error // Remove removes transaction and related signatures from storage if not in finalized or errored state Remove(sig solana.Signature) (string, error) // ListAll returns all of the signatures being tracked for all transactions not yet finalized or errored @@ -51,7 +50,7 @@ type PendingTxContext interface { // GetSignatureInfo returns the transaction ID and TxState for the provided signature GetSignatureInfo(sig solana.Signature) (txInfo, error) // OnReorg resets the transaction state to Broadcasted for the given signature and returns the pendingTx for retrying. - OnReorg(sig solana.Signature) (pendingTx, retryCtx, error) + OnReorg(sig solana.Signature) (pendingTx, error) } // finishedTx is used to store info required to track transactions to finality or error @@ -75,16 +74,11 @@ type txInfo struct { state TxState } -type retryCtx struct { - ctx context.Context - cancel context.CancelFunc -} - var _ PendingTxContext = &pendingTxContext{} type pendingTxContext struct { - sigToRetryCtx map[solana.Signature]retryCtx - sigToTxInfo map[solana.Signature]txInfo + cancelBy map[string]context.CancelFunc + sigToTxInfo map[solana.Signature]txInfo broadcastedProcessedTxs map[string]pendingTx // broadcasted and processed transactions that may require retry and bumping confirmedTxs map[string]pendingTx // transactions that require monitoring for re-org @@ -95,8 +89,8 @@ type pendingTxContext struct { func newPendingTxContext() *pendingTxContext { return &pendingTxContext{ - sigToRetryCtx: map[solana.Signature]retryCtx{}, - sigToTxInfo: map[solana.Signature]txInfo{}, + cancelBy: map[string]context.CancelFunc{}, + sigToTxInfo: map[solana.Signature]txInfo{}, broadcastedProcessedTxs: map[string]pendingTx{}, confirmedTxs: map[string]pendingTx{}, @@ -104,9 +98,12 @@ func newPendingTxContext() *pendingTxContext { } } -// New adds a new tranasction in Broadcasted state to the storage -func (c *pendingTxContext) New(tx pendingTx) error { +func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel context.CancelFunc) error { err := c.withReadLock(func() error { + // validate signature does not exist + if _, exists := c.sigToTxInfo[sig]; exists { + return ErrSigAlreadyExists + } // validate id does not exist if _, exists := c.broadcastedProcessedTxs[tx.id]; exists { return ErrIDAlreadyExists @@ -117,12 +114,19 @@ func (c *pendingTxContext) New(tx pendingTx) error { return err } - // upgrade to write lock if id do not exist + // upgrade to write lock if sig or id do not exist _, err = c.withWriteLock(func() (string, error) { + if _, exists := c.sigToTxInfo[sig]; exists { + return "", ErrSigAlreadyExists + } if _, exists := c.broadcastedProcessedTxs[tx.id]; exists { return "", ErrIDAlreadyExists } - tx.signatures = []solana.Signature{} + // save cancel func + c.cancelBy[tx.id] = cancel + c.sigToTxInfo[sig] = txInfo{id: tx.id, state: Broadcasted} + // add signature to tx + tx.signatures = append(tx.signatures, sig) tx.createTs = time.Now() // save to the broadcasted map since transaction was just broadcasted c.broadcastedProcessedTxs[tx.id] = tx @@ -131,9 +135,7 @@ func (c *pendingTxContext) New(tx pendingTx) error { return err } -// AddSignature adds a new signature to a broadcasted transaction in the pending transaction context. -// Additionally, it associates the provided context and cancel function with the signature to manage retry and bumping cycles. -func (c *pendingTxContext) AddSignature(ctx context.Context, cancel context.CancelFunc, id string, sig solana.Signature) error { +func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { err := c.withReadLock(func() error { // signature already exists if _, exists := c.sigToTxInfo[sig]; exists { @@ -162,8 +164,6 @@ func (c *pendingTxContext) AddSignature(ctx context.Context, cancel context.Canc tx := c.broadcastedProcessedTxs[id] // save new signature tx.signatures = append(tx.signatures, sig) - // save retryCtx to stop retry/bumping cycles - c.sigToRetryCtx[sig] = retryCtx{ctx: ctx, cancel: cancel} // save updated tx to broadcasted map c.broadcastedProcessedTxs[id] = tx return "", nil @@ -208,13 +208,15 @@ func (c *pendingTxContext) Remove(sig solana.Signature) (id string, err error) { delete(c.confirmedTxs, info.id) } - // remove all signatures associated with transaction from sig map and cancel any associated contexts to stop retry/bumping cycles + // call cancel func + remove from map + if cancel, exists := c.cancelBy[info.id]; exists { + cancel() // cancel context + delete(c.cancelBy, info.id) + } + + // remove all signatures associated with transaction from sig map for _, s := range tx.signatures { delete(c.sigToTxInfo, s) - if rtryCtx, exists := c.sigToRetryCtx[s]; exists { - rtryCtx.cancel() - delete(c.sigToRetryCtx, s) - } } return info.id, nil }) @@ -342,9 +344,9 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { return info.id, ErrTransactionNotFound } // call cancel func + remove from map to stop the retry/bumping cycle for this transaction - if rtryCtx, exists := c.sigToRetryCtx[sig]; exists { - rtryCtx.cancel() - delete(c.sigToRetryCtx, sig) + if cancel, exists := c.cancelBy[info.id]; exists { + cancel() // cancel context + delete(c.cancelBy, info.id) } // update sig and tx state to Confirmed info.state = Confirmed @@ -392,18 +394,19 @@ func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout ti if !broadcastedExists && !confirmedExists { return info.id, ErrTransactionNotFound } + // call cancel func + remove from map to stop the retry/bumping cycle for this transaction + // cancel is expected to be called and removed when tx is confirmed but checked here too in case state is skipped + if cancel, exists := c.cancelBy[info.id]; exists { + cancel() // cancel context + delete(c.cancelBy, info.id) + } // delete from broadcasted map, if exists delete(c.broadcastedProcessedTxs, info.id) // delete from confirmed map, if exists delete(c.confirmedTxs, info.id) // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic - // call cancel func + remove from map to stop the retry/bumping cycle for this transaction for _, s := range tx.signatures { delete(c.sigToTxInfo, s) - if rtryCtx, exists := c.sigToRetryCtx[s]; exists { - rtryCtx.cancel() - delete(c.sigToRetryCtx, s) - } } // if retention duration is set to 0, delete transaction from storage // otherwise, move to finalized map @@ -500,18 +503,18 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D if !broadcastedExists && !confirmedExists { return "", ErrTransactionNotFound } + // call cancel func + remove from map + if cancel, exists := c.cancelBy[info.id]; exists { + cancel() // cancel context + delete(c.cancelBy, info.id) + } // delete from broadcasted map, if exists delete(c.broadcastedProcessedTxs, info.id) // delete from confirmed map, if exists delete(c.confirmedTxs, info.id) // remove all related signatures from the sigToTxInfo map to skip picking up this tx in the confirmation logic - // call cancel func + remove from map to stop the retry/bumping cycle for this transaction for _, s := range tx.signatures { delete(c.sigToTxInfo, s) - if rtryCtx, exists := c.sigToRetryCtx[s]; exists { - rtryCtx.cancel() // cancel context - delete(c.sigToRetryCtx, s) - } } // if retention duration is set to 0, skip adding transaction to the errored map if retentionTimeout == 0 { @@ -618,7 +621,7 @@ func (c *pendingTxContext) GetSignatureInfo(sig solana.Signature) (txInfo, error return info, nil } -func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, retryCtx, error) { +func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { // Acquire a read lock to check if the signature exists and needs to be reset err := c.withReadLock(func() error { // Check if the signature is still being tracked @@ -637,11 +640,10 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, retryCtx, e }) if err != nil { // If transaction or sig are not found, return - return pendingTx{}, retryCtx{}, err + return pendingTx{}, err } var pTx pendingTx - var rtryCtx retryCtx // Acquire a write lock to perform the state reset _, err = c.withWriteLock(func() (string, error) { // Retrieve sig and tx again inside the write lock @@ -650,12 +652,6 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, retryCtx, e return "", ErrSigDoesNotExist } - // Check if the retryCtx is still relevant - rtryCtx, exists = c.sigToRetryCtx[sig] - if !exists { - return "", fmt.Errorf("retry context not found for signature %s associated to tx id %s", sig.String(), info.id) - } - // Attempt to find the transaction in the broadcasted or confirmed maps var tx pendingTx var broadcastedExists, confirmedExists bool @@ -677,11 +673,11 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, retryCtx, e }) if err != nil { // If transaction or sig were not found, return - return pendingTx{}, rtryCtx, err + return pendingTx{}, err } // Return the transaction for retrying - return pTx, rtryCtx, nil + return pTx, nil } func (c *pendingTxContext) withReadLock(fn func() error) error { @@ -721,12 +717,12 @@ func newPendingTxContextWithProm(id string) *pendingTxContextWithProm { } } -func (c *pendingTxContextWithProm) New(msg pendingTx) error { - return c.pendingTx.New(msg) +func (c *pendingTxContextWithProm) New(msg pendingTx, sig solana.Signature, cancel context.CancelFunc) error { + return c.pendingTx.New(msg, sig, cancel) } -func (c *pendingTxContextWithProm) AddSignature(ctx context.Context, cancel context.CancelFunc, id string, sig solana.Signature) error { - return c.pendingTx.AddSignature(ctx, cancel, id, sig) +func (c *pendingTxContextWithProm) AddSignature(id string, sig solana.Signature) error { + return c.pendingTx.AddSignature(id, sig) } func (c *pendingTxContextWithProm) OnProcessed(sig solana.Signature) (string, error) { @@ -815,6 +811,6 @@ func (c *pendingTxContextWithProm) GetSignatureInfo(sig solana.Signature) (txInf return c.pendingTx.GetSignatureInfo(sig) } -func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, retryCtx, error) { +func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, error) { return c.pendingTx.OnReorg(sig) } diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 762630d52..5baaf23a0 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -40,15 +40,13 @@ func TestPendingTxContext_add_remove_multiple(t *testing.T) { for i := 0; i < n; i++ { sig, cancel := newProcess() msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - assert.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) assert.NoError(t, err) ids[sig] = msg.id } // cannot add signature for non existent ID - require.Error(t, txs.AddSignature(ctx, func() {}, uuid.New().String(), solana.Signature{})) + require.Error(t, txs.AddSignature(uuid.New().String(), solana.Signature{})) // return list of signatures list := txs.ListAll() @@ -71,15 +69,13 @@ func TestPendingTxContext_add_remove_multiple(t *testing.T) { func TestPendingTxContext_new(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) sig := randomSignature(t) txs := newPendingTxContext() // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Check it exists in signature map @@ -107,7 +103,7 @@ func TestPendingTxContext_new(t *testing.T) { func TestPendingTxContext_add_signature(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() t.Run("successfully add signature to transaction", func(t *testing.T) { @@ -116,12 +112,10 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig1) + err := txs.New(msg, sig1, cancel) require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig2) + err = txs.AddSignature(msg.id, sig2) require.NoError(t, err) // Check signature map @@ -153,12 +147,10 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err = txs.AddSignature(msg.id, sig) require.ErrorIs(t, err, ErrSigAlreadyExists) }) @@ -168,12 +160,10 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig1) + err := txs.New(msg, sig1, cancel) require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, "bad id", sig2) + err = txs.AddSignature("bad id", sig2) require.ErrorIs(t, err, ErrTransactionNotFound) }) @@ -183,9 +173,7 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig1) + err := txs.New(msg, sig1, cancel) require.NoError(t, err) // Transition to processed state @@ -198,14 +186,14 @@ func TestPendingTxContext_add_signature(t *testing.T) { require.NoError(t, err) require.Equal(t, msg.id, id) - err = txs.AddSignature(ctx, cancel, msg.id, sig2) + err = txs.AddSignature(msg.id, sig2) require.ErrorIs(t, err, ErrTransactionNotFound) }) } func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -214,9 +202,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to processed state @@ -252,9 +238,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to processed state @@ -277,9 +261,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to processed state @@ -307,9 +289,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to errored state @@ -327,9 +307,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to processed state @@ -345,7 +323,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { func TestPendingTxContext_on_confirmed(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -354,9 +332,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to processed state @@ -397,9 +373,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to processed state @@ -427,9 +401,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to errored state @@ -447,9 +419,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to processed state @@ -470,7 +440,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { func TestPendingTxContext_on_finalized(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -480,13 +450,11 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig1) + err := txs.New(msg, sig1, cancel) require.NoError(t, err) // Add second signature - err = txs.AddSignature(ctx, cancel, msg.id, sig2) + err = txs.AddSignature(msg.id, sig2) require.NoError(t, err) // Transition to finalized state @@ -522,13 +490,11 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig1) + err := txs.New(msg, sig1, cancel) require.NoError(t, err) // Add second signature - err = txs.AddSignature(ctx, cancel, msg.id, sig2) + err = txs.AddSignature(msg.id, sig2) require.NoError(t, err) // Transition to processed state @@ -573,9 +539,7 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig1) + err := txs.New(msg, sig1, cancel) require.NoError(t, err) // Transition to processed state @@ -615,9 +579,7 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to errored state @@ -633,7 +595,7 @@ func TestPendingTxContext_on_finalized(t *testing.T) { func TestPendingTxContext_on_error(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -642,9 +604,7 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to errored state @@ -677,9 +637,7 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to errored state @@ -717,9 +675,7 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to fatally errored state @@ -748,9 +704,7 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to errored state @@ -785,9 +739,7 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to confirmed state @@ -804,7 +756,7 @@ func TestPendingTxContext_on_error(t *testing.T) { func TestPendingTxContext_on_prebroadcast_error(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -845,9 +797,7 @@ func TestPendingTxContext_on_prebroadcast_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} // Add transaction to broadcasted map - err := txs.New(msg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) require.NoError(t, err) // Transition to errored state @@ -870,7 +820,7 @@ func TestPendingTxContext_on_prebroadcast_error(t *testing.T) { func TestPendingTxContext_remove(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -884,18 +834,14 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new broadcasted transaction with extra sig broadcastedMsg := pendingTx{id: uuid.NewString()} - err := txs.New(broadcastedMsg) + err := txs.New(broadcastedMsg, broadcastedSig1, cancel) require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, broadcastedMsg.id, broadcastedSig1) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, broadcastedMsg.id, broadcastedSig2) + err = txs.AddSignature(broadcastedMsg.id, broadcastedSig2) require.NoError(t, err) // Create new processed transaction processedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(processedMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, processedMsg.id, processedSig) + err = txs.New(processedMsg, processedSig, cancel) require.NoError(t, err) id, err := txs.OnProcessed(processedSig) require.NoError(t, err) @@ -903,9 +849,7 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new confirmed transaction confirmedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(confirmedMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, confirmedMsg.id, confirmedSig) + err = txs.New(confirmedMsg, confirmedSig, cancel) require.NoError(t, err) id, err = txs.OnConfirmed(confirmedSig) require.NoError(t, err) @@ -913,9 +857,7 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new finalized transaction finalizedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(finalizedMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, finalizedMsg.id, finalizedSig) + err = txs.New(finalizedMsg, finalizedSig, cancel) require.NoError(t, err) id, err = txs.OnFinalized(finalizedSig, retentionTimeout) require.NoError(t, err) @@ -923,9 +865,7 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new errored transaction erroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(erroredMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, erroredMsg.id, erroredSig) + err = txs.New(erroredMsg, erroredSig, cancel) require.NoError(t, err) id, err = txs.OnError(erroredSig, retentionTimeout, Errored, 0) require.NoError(t, err) @@ -979,7 +919,6 @@ func TestPendingTxContext_remove(t *testing.T) { // Check sig list is empty after all removals require.Empty(t, txs.ListAll()) } - func TestPendingTxContext_trim_finalized_errored_txs(t *testing.T) { t.Parallel() txs := newPendingTxContext() @@ -1017,14 +956,12 @@ func TestPendingTxContext_trim_finalized_errored_txs(t *testing.T) { func TestPendingTxContext_expired(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) sig := solana.Signature{} txs := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg) - assert.NoError(t, err) - err = txs.AddSignature(ctx, cancel, msg.id, sig) + err := txs.New(msg, sig, cancel) assert.NoError(t, err) msg, exists := txs.broadcastedProcessedTxs[msg.id] @@ -1047,17 +984,16 @@ func TestPendingTxContext_expired(t *testing.T) { func TestPendingTxContext_race(t *testing.T) { t.Run("new", func(t *testing.T) { txCtx := newPendingTxContext() - txID := uuid.NewString() var wg sync.WaitGroup wg.Add(2) var err [2]error go func() { - err[0] = txCtx.New(pendingTx{id: txID}) + err[0] = txCtx.New(pendingTx{id: uuid.NewString()}, solana.Signature{}, func() {}) wg.Done() }() go func() { - err[1] = txCtx.New(pendingTx{id: txID}) + err[1] = txCtx.New(pendingTx{id: uuid.NewString()}, solana.Signature{}, func() {}) wg.Done() }() @@ -1068,19 +1004,18 @@ func TestPendingTxContext_race(t *testing.T) { t.Run("add signature", func(t *testing.T) { txCtx := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - createErr := txCtx.New(msg) + createErr := txCtx.New(msg, solana.Signature{}, func() {}) require.NoError(t, createErr) - ctx, cancel := context.WithCancel(tests.Context(t)) var wg sync.WaitGroup wg.Add(2) var err [2]error go func() { - err[0] = txCtx.AddSignature(ctx, cancel, msg.id, solana.Signature{1}) + err[0] = txCtx.AddSignature(msg.id, solana.Signature{1}) wg.Done() }() go func() { - err[1] = txCtx.AddSignature(ctx, cancel, msg.id, solana.Signature{1}) + err[1] = txCtx.AddSignature(msg.id, solana.Signature{1}) wg.Done() }() @@ -1091,7 +1026,7 @@ func TestPendingTxContext_race(t *testing.T) { t.Run("remove", func(t *testing.T) { txCtx := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - err := txCtx.New(msg) + err := txCtx.New(msg, solana.Signature{}, func() {}) require.NoError(t, err) var wg sync.WaitGroup wg.Add(2) @@ -1111,7 +1046,7 @@ func TestPendingTxContext_race(t *testing.T) { func TestGetTxState(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(tests.Context(t)) + _, cancel := context.WithCancel(tests.Context(t)) txs := newPendingTxContext() retentionTimeout := 5 * time.Second @@ -1124,17 +1059,13 @@ func TestGetTxState(t *testing.T) { // Create new broadcasted transaction with extra sig broadcastedMsg := pendingTx{id: uuid.NewString()} - err := txs.New(broadcastedMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, broadcastedMsg.id, broadcastedSig) + err := txs.New(broadcastedMsg, broadcastedSig, cancel) require.NoError(t, err) var state TxState // Create new processed transaction processedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(processedMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, processedMsg.id, processedSig) + err = txs.New(processedMsg, processedSig, cancel) require.NoError(t, err) id, err := txs.OnProcessed(processedSig) require.NoError(t, err) @@ -1146,9 +1077,7 @@ func TestGetTxState(t *testing.T) { // Create new confirmed transaction confirmedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(confirmedMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, confirmedMsg.id, confirmedSig) + err = txs.New(confirmedMsg, confirmedSig, cancel) require.NoError(t, err) id, err = txs.OnConfirmed(confirmedSig) require.NoError(t, err) @@ -1160,9 +1089,7 @@ func TestGetTxState(t *testing.T) { // Create new finalized transaction finalizedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(finalizedMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, finalizedMsg.id, finalizedSig) + err = txs.New(finalizedMsg, finalizedSig, cancel) require.NoError(t, err) id, err = txs.OnFinalized(finalizedSig, retentionTimeout) require.NoError(t, err) @@ -1174,9 +1101,7 @@ func TestGetTxState(t *testing.T) { // Create new errored transaction erroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(erroredMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, erroredMsg.id, erroredSig) + err = txs.New(erroredMsg, erroredSig, cancel) require.NoError(t, err) id, err = txs.OnError(erroredSig, retentionTimeout, Errored, 0) require.NoError(t, err) @@ -1188,9 +1113,7 @@ func TestGetTxState(t *testing.T) { // Create new fatally errored transaction fatallyErroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(fatallyErroredMsg) - require.NoError(t, err) - err = txs.AddSignature(ctx, cancel, fatallyErroredMsg.id, fatallyErroredSig) + err = txs.New(fatallyErroredMsg, fatallyErroredSig, cancel) require.NoError(t, err) id, err = txs.OnError(fatallyErroredSig, retentionTimeout, FatallyErrored, 0) require.NoError(t, err) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 77c354771..d46d8c38b 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -216,16 +216,10 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("tx failed initial transmit: %w", errors.Join(initSendErr, stateTransitionErr)) } - // Create new transaction in memory - if err := txm.txs.New(msg); err != nil { - cancel() - return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to create new transaction in memory: %w", err) - } - - // Associate signature and retryCtx to tx - if err := txm.txs.AddSignature(ctx, cancel, msg.id, sig); err != nil { - cancel() - return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to save initial signature (%s) to inflight txs: %w", sig, err) + // Store tx signature and cancel function + if err := txm.txs.New(msg, sig, cancel); err != nil { + cancel() // Cancel context when exiting early + return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to save tx signature (%s) to inflight txs: %w", sig, err) } txm.lggr.Debugw("tx initial broadcast", "id", msg.id, "fee", msg.cfg.BaseComputeUnitPrice, "signature", sig) @@ -234,7 +228,7 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(ctx, cancel, msg, initTx, sig) + txm.retryTx(ctx, msg, initTx, sig, func() {}) }() // Return signed tx, id, signature for use in simulation @@ -285,7 +279,8 @@ func (txm *Txm) buildTx(ctx context.Context, msg pendingTx, retryCount int) (sol // retryTx contains the logic for retrying the transaction, including exponential backoff and fee bumping. // Retries until context cancelled by timeout or called externally. // It uses handleRetry helper function to handle each retry attempt. -func (txm *Txm) retryTx(ctx context.Context, cancel context.CancelFunc, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature) { +func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature, cancel context.CancelFunc) { + defer cancel() // Initialize signature list with initialTx signature. This list will be used to add new signatures and track retry attempts. sigs := &signatureList{} sigs.Allocate() @@ -334,7 +329,7 @@ func (txm *Txm) retryTx(ctx context.Context, cancel context.CancelFunc, msg pend wg.Add(1) go func(bump bool, count int, retryTx solanaGo.Transaction) { defer wg.Done() - txm.handleRetry(ctx, cancel, msg, bump, count, retryTx, sigs) + txm.handleRetry(ctx, msg, bump, count, retryTx, sigs) }(shouldBump, bumpCount, currentTx) } @@ -348,7 +343,7 @@ func (txm *Txm) retryTx(ctx context.Context, cancel context.CancelFunc, msg pend } // handleRetry handles the logic for each retry attempt, including sending the transaction, updating signatures, and logging. -func (txm *Txm) handleRetry(ctx context.Context, cancel context.CancelFunc, msg pendingTx, bump bool, count int, retryTx solanaGo.Transaction, sigs *signatureList) { +func (txm *Txm) handleRetry(ctx context.Context, msg pendingTx, bump bool, count int, retryTx solanaGo.Transaction, sigs *signatureList) { // send retry transaction retrySig, err := txm.sendTx(ctx, &retryTx) if err != nil { @@ -363,7 +358,7 @@ func (txm *Txm) handleRetry(ctx context.Context, cancel context.CancelFunc, msg // if bump is true, update signature list and set new signature in space already allocated. if bump { - if err := txm.txs.AddSignature(ctx, cancel, msg.id, retrySig); err != nil { + if err := txm.txs.AddSignature(msg.id, retrySig); err != nil { txm.lggr.Warnw("error in adding retry transaction", "error", err, "id", msg.id) return } @@ -547,17 +542,16 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status if isStatusRegression(txInfo.state, currentTxState) { txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) // Handle reorg in our in memory layer and retry transaction - // We'll retrieve the associated pendgingTx and retryCtx to the sig - // attempting to restart the retry/bumping cycle for it if it's still in-flight - pTx, retryCtx, err := txm.txs.OnReorg(sig) + pTx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) return err } + retryCtx, cancel := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: How should we handle the ctx? txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(retryCtx.ctx, retryCtx.cancel, pTx, pTx.tx, sig) + txm.retryTx(retryCtx, pTx, pTx.tx, sig, cancel) }() } From d1f1ae724128f692f062256e7ea1655296b06c9e Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 13:29:03 -0300 Subject: [PATCH 16/31] Revert "fix listAllExpiredBroadcastedTxs" This reverts commit f4c6069a6818bba90c73aed28b07909aac4859ea. --- pkg/solana/txm/pendingtx.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index f8b76830f..722258659 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -233,22 +233,13 @@ func (c *pendingTxContext) ListAll() []solana.Signature { func (c *pendingTxContext) ListAllExpiredBroadcastedTxs(currHeight uint64) []pendingTx { c.lock.RLock() defer c.lock.RUnlock() - - expiredTxes := make([]pendingTx, 0, len(c.broadcastedProcessedTxs)) // worst case, all of them - - for id, tx := range c.broadcastedProcessedTxs { - state, err := c.GetTxState(id) - if err != nil { - continue // Ignore transactions that are not found - } - - // Check if the transaction is still in the Broadcasted state - if state == Broadcasted && tx.lastValidBlockHeight < currHeight { - expiredTxes = append(expiredTxes, tx) + broadcastedTxes := make([]pendingTx, 0, len(c.broadcastedProcessedTxs)) // worst case, all of them + for _, tx := range c.broadcastedProcessedTxs { + if tx.lastValidBlockHeight < currHeight { + broadcastedTxes = append(broadcastedTxes, tx) } } - - return expiredTxes + return broadcastedTxes } // Expired returns if the timeout for trying to confirm a signature has been reached From 8911df24aafd1cef86bc34244db101ca091ae9d0 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 13:30:57 -0300 Subject: [PATCH 17/31] Revert "move state from txes to sigs" This reverts commit 3a6e643c35b0a31332713099dded6b811226f692. --- pkg/solana/txm/pendingtx.go | 80 ++++++++--------------------- pkg/solana/txm/pendingtx_test.go | 6 +-- pkg/solana/txm/txm.go | 9 ++-- pkg/solana/txm/txm_internal_test.go | 5 +- 4 files changed, 32 insertions(+), 68 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 722258659..9d6be7428 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -60,6 +60,7 @@ type pendingTx struct { signatures []solana.Signature id string createTs time.Time + state TxState lastValidBlockHeight uint64 // to track expiration } @@ -128,6 +129,7 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex // add signature to tx tx.signatures = append(tx.signatures, sig) tx.createTs = time.Now() + tx.state = Broadcasted // save to the broadcasted map since transaction was just broadcasted c.broadcastedProcessedTxs[tx.id] = tx return "", nil @@ -235,7 +237,7 @@ func (c *pendingTxContext) ListAllExpiredBroadcastedTxs(currHeight uint64) []pen defer c.lock.RUnlock() broadcastedTxes := make([]pendingTx, 0, len(c.broadcastedProcessedTxs)) // worst case, all of them for _, tx := range c.broadcastedProcessedTxs { - if tx.lastValidBlockHeight < currHeight { + if tx.state == Broadcasted && tx.lastValidBlockHeight < currHeight { broadcastedTxes = append(broadcastedTxes, tx) } } @@ -271,12 +273,12 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { return ErrSigDoesNotExist } // Transactions should only move to processed from broadcasted - _, exists := c.broadcastedProcessedTxs[info.id] + tx, exists := c.broadcastedProcessedTxs[info.id] if !exists { return ErrTransactionNotFound } - // Check if sig already in processed state - if info.state == Processed { + // Check if tranasction already in processed state + if tx.state == Processed { return ErrAlreadyInExpectedState } return nil @@ -295,8 +297,8 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { if !exists { return info.id, ErrTransactionNotFound } - // update sig to Processed - info.state = Processed + // update sig and tx to Processed + info.state, tx.state = Processed, Processed // save updated sig and tx back to the maps c.sigToTxInfo[sig] = info return info.id, nil @@ -310,8 +312,8 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { if !sigExists { return ErrSigDoesNotExist } - // Check if sig already in confirmed state - if _, exists := c.confirmedTxs[info.id]; exists && info.state == Confirmed { + // Check if transaction already in confirmed state + if tx, exists := c.confirmedTxs[info.id]; exists && tx.state == Confirmed { return ErrAlreadyInExpectedState } // Transactions should only move to confirmed from broadcasted/processed @@ -340,7 +342,7 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) { delete(c.cancelBy, info.id) } // update sig and tx state to Confirmed - info.state = Confirmed + info.state, tx.state = Confirmed, Confirmed c.sigToTxInfo[sig] = info // move tx to confirmed map c.confirmedTxs[info.id] = tx @@ -521,58 +523,21 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D }) } -// GetTxState retrieves the aggregated state of a transaction based on all its signatures. -// It performs state aggregation only for transactions in broadcastedProcessedTxs or confirmedTxs. -// For transactions in finalizedErroredTxs, it directly returns the stored state. func (c *pendingTxContext) GetTxState(id string) (TxState, error) { c.lock.RLock() defer c.lock.RUnlock() - - // Check if the transaction exists in broadcastedProcessedTxs if tx, exists := c.broadcastedProcessedTxs[id]; exists { - return c.aggregateTxState(tx), nil + return tx.state, nil } - - // Check if the transaction exists in confirmedTxs if tx, exists := c.confirmedTxs[id]; exists { - return c.aggregateTxState(tx), nil + return tx.state, nil } - - // Check if the transaction exists in finalizedErroredTxs if tx, exists := c.finalizedErroredTxs[id]; exists { return tx.state, nil } - - // Transaction not found in any map return NotFound, fmt.Errorf("failed to find transaction for id: %s", id) } -// aggregateTxState determines the highest TxState among all signatures of a pending transaction. -func (c *pendingTxContext) aggregateTxState(tx pendingTx) TxState { - // Define the priority of states - statePriority := map[TxState]int{ - Broadcasted: 1, - Processed: 2, - Confirmed: 3, - } - - // Update highestState based on individual signature states - highestState := Broadcasted - for _, sig := range tx.signatures { - info, exists := c.sigToTxInfo[sig] - if !exists { - continue - } - if priority, ok := statePriority[info.state]; ok { - if priority > statePriority[highestState] { - highestState = info.state - } - } - } - - return highestState -} - // TrimFinalizedErroredTxs deletes transactions from the finalized/errored map and the allTxs map after the retention period has passed func (c *pendingTxContext) TrimFinalizedErroredTxs() int { var expiredIDs []string @@ -621,11 +586,12 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { return ErrSigDoesNotExist } - // Check if the transaction is still in a non-finalized/non-errored state - if _, exists := c.broadcastedProcessedTxs[info.id]; !exists { - if _, exists := c.confirmedTxs[info.id]; !exists { - return ErrTransactionNotFound - } + // Check if the transaction is still in a non finalized/errored state + var broadcastedExists, confirmedExists bool + _, broadcastedExists = c.broadcastedProcessedTxs[info.id] + _, confirmedExists = c.confirmedTxs[info.id] + if !broadcastedExists && !confirmedExists { + return ErrTransactionNotFound } return nil }) @@ -642,8 +608,6 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { if !exists { return "", ErrSigDoesNotExist } - - // Attempt to find the transaction in the broadcasted or confirmed maps var tx pendingTx var broadcastedExists, confirmedExists bool if tx, broadcastedExists = c.broadcastedProcessedTxs[info.id]; broadcastedExists { @@ -653,12 +617,12 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { pTx = tx } if !broadcastedExists && !confirmedExists { - // transaction does not exist in any non finalized/errored maps + // transcation does not exist in any non finalized/errored maps return "", ErrTransactionNotFound } - // Reset the signature status for retrying - info.state = Broadcasted + // Reset the signature status and tx for retrying + info.state, pTx.state = Broadcasted, Broadcasted c.sigToTxInfo[sig] = info return "", nil }) diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 5baaf23a0..60b208412 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -90,7 +90,7 @@ func TestPendingTxContext_new(t *testing.T) { require.Equal(t, sig, tx.signatures[0]) // Check status is Broadcasted - require.Equal(t, Broadcasted, txInfo.state) + require.Equal(t, Broadcasted, tx.state) // Check it does not exist in confirmed map _, exists = txs.confirmedTxs[msg.id] @@ -222,7 +222,7 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { require.Equal(t, sig, tx.signatures[0]) // Check status is Processed - require.Equal(t, Processed, txInfo.state) + require.Equal(t, Processed, tx.state) // Check it does not exist in confirmed map _, exists = txs.confirmedTxs[msg.id] @@ -361,7 +361,7 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { require.Equal(t, sig, tx.signatures[0]) // Check status is Confirmed - require.Equal(t, Confirmed, txInfo.state) + require.Equal(t, Confirmed, tx.state) // Check it does not exist in finalized map _, exists = txs.finalizedErroredTxs[msg.id] diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index d46d8c38b..b0e63f513 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -228,7 +228,7 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(ctx, msg, initTx, sig, func() {}) + txm.retryTx(ctx, msg, initTx, sig) }() // Return signed tx, id, signature for use in simulation @@ -279,8 +279,7 @@ func (txm *Txm) buildTx(ctx context.Context, msg pendingTx, retryCount int) (sol // retryTx contains the logic for retrying the transaction, including exponential backoff and fee bumping. // Retries until context cancelled by timeout or called externally. // It uses handleRetry helper function to handle each retry attempt. -func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature, cancel context.CancelFunc) { - defer cancel() +func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature) { // Initialize signature list with initialTx signature. This list will be used to add new signatures and track retry attempts. sigs := &signatureList{} sigs.Allocate() @@ -547,11 +546,11 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) return err } - retryCtx, cancel := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: How should we handle the ctx? + retryCtx, _ := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: Ask here. How should we handle the ctx? txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(retryCtx, pTx, pTx.tx, sig, cancel) + txm.retryTx(retryCtx, pTx, pTx.tx, sig) }() } diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index b835d59f0..4f268aed2 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1503,7 +1503,8 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { return sig1, nil } - // There will be no rebroadcasts txes to process + // Mock LatestBlockhash to return an invalid blockhash less than slotHeight + // We won't use it as there will be no rebroadcasts txes to process. All txes will be confirmed before. slotHeightFunc := func() (uint64, error) { return uint64(1500), nil } @@ -1513,7 +1514,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { defer func() { callCount++ }() return &rpc.GetLatestBlockhashResult{ Value: &rpc.LatestBlockhashResult{ - LastValidBlockHeight: uint64(2000), + LastValidBlockHeight: uint64(1000), }, }, nil } From 52ce0e98b227e75f90a8208dacb86db477b9c833 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 13:50:25 -0300 Subject: [PATCH 18/31] fix tx state --- pkg/solana/txm/pendingtx.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 9d6be7428..cba91c52f 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -293,7 +293,7 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { if !sigExists { return info.id, ErrSigDoesNotExist } - _, exists := c.broadcastedProcessedTxs[info.id] + tx, exists := c.broadcastedProcessedTxs[info.id] if !exists { return info.id, ErrTransactionNotFound } @@ -301,6 +301,7 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) { info.state, tx.state = Processed, Processed // save updated sig and tx back to the maps c.sigToTxInfo[sig] = info + c.broadcastedProcessedTxs[info.id] = tx return info.id, nil }) } From fbbe978e8dfa73f629d29653b7e493d54f7f3460 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 18:02:36 -0300 Subject: [PATCH 19/31] address feedback --- pkg/solana/txm/pendingtx.go | 72 +++++------ pkg/solana/txm/pendingtx_test.go | 178 ++++++++++++++++++++-------- pkg/solana/txm/txm.go | 47 +++++--- pkg/solana/txm/txm_internal_test.go | 4 +- pkg/solana/txm/utils.go | 51 ++++---- 5 files changed, 225 insertions(+), 127 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index cba91c52f..b39b6b0dc 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -20,10 +20,11 @@ var ( ) type PendingTxContext interface { - // New adds a new tranasction in Broadcasted state to the storage - New(msg pendingTx, sig solana.Signature, cancel context.CancelFunc) error - // AddSignature adds a new signature for an existing transaction ID - AddSignature(id string, sig solana.Signature) error + // New adds a new transaction in Broadcasted state to the storage + New(msg pendingTx) error + // AddSignature adds a new signature to a broadcasted transaction in the pending transaction context. + // It associates the provided context and cancel function with the signature to manage retry and bumping cycles. + AddSignature(cancel context.CancelFunc, id string, sig solana.Signature) error // Remove removes transaction and related signatures from storage if not in finalized or errored state Remove(sig solana.Signature) (string, error) // ListAll returns all of the signatures being tracked for all transactions not yet finalized or errored @@ -99,12 +100,8 @@ func newPendingTxContext() *pendingTxContext { } } -func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel context.CancelFunc) error { +func (c *pendingTxContext) New(tx pendingTx) error { err := c.withReadLock(func() error { - // validate signature does not exist - if _, exists := c.sigToTxInfo[sig]; exists { - return ErrSigAlreadyExists - } // validate id does not exist if _, exists := c.broadcastedProcessedTxs[tx.id]; exists { return ErrIDAlreadyExists @@ -115,19 +112,12 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex return err } - // upgrade to write lock if sig or id do not exist + // upgrade to write lock if id do not exist _, err = c.withWriteLock(func() (string, error) { - if _, exists := c.sigToTxInfo[sig]; exists { - return "", ErrSigAlreadyExists - } if _, exists := c.broadcastedProcessedTxs[tx.id]; exists { return "", ErrIDAlreadyExists } - // save cancel func - c.cancelBy[tx.id] = cancel - c.sigToTxInfo[sig] = txInfo{id: tx.id, state: Broadcasted} - // add signature to tx - tx.signatures = append(tx.signatures, sig) + tx.signatures = []solana.Signature{} tx.createTs = time.Now() tx.state = Broadcasted // save to the broadcasted map since transaction was just broadcasted @@ -137,7 +127,7 @@ func (c *pendingTxContext) New(tx pendingTx, sig solana.Signature, cancel contex return err } -func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { +func (c *pendingTxContext) AddSignature(cancel context.CancelFunc, id string, sig solana.Signature) error { err := c.withReadLock(func() error { // signature already exists if _, exists := c.sigToTxInfo[sig]; exists { @@ -168,6 +158,12 @@ func (c *pendingTxContext) AddSignature(id string, sig solana.Signature) error { tx.signatures = append(tx.signatures, sig) // save updated tx to broadcasted map c.broadcastedProcessedTxs[id] = tx + // set cancel context if not already set to handle reorgs when regressing from confirmed state + // previous context was removed so we associate a new context to our transaction to restart the retry/bumping cycle + if _, exists := c.cancelBy[id]; !exists { + c.cancelBy[id] = cancel + } + return "", nil }) return err @@ -609,30 +605,40 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { if !exists { return "", ErrSigDoesNotExist } - var tx pendingTx - var broadcastedExists, confirmedExists bool - if tx, broadcastedExists = c.broadcastedProcessedTxs[info.id]; broadcastedExists { + var broadcastedProcessedExists, confirmedExists bool + if tx, broadcastedProcessedExists := c.broadcastedProcessedTxs[info.id]; broadcastedProcessedExists { pTx = tx } - if tx, confirmedExists = c.confirmedTxs[info.id]; confirmedExists { + if tx, confirmedExists := c.confirmedTxs[info.id]; confirmedExists { pTx = tx } - if !broadcastedExists && !confirmedExists { - // transcation does not exist in any non finalized/errored maps + + if !broadcastedProcessedExists && !confirmedExists { + // transaction does not exist in any non finalized/errored maps return "", ErrTransactionNotFound } - // Reset the signature status and tx for retrying - info.state, pTx.state = Broadcasted, Broadcasted + // If the transaction regressed from processed state, we only need to reset the state + if broadcastedProcessedExists { + info.state, pTx.state = Broadcasted, Broadcasted + c.sigToTxInfo[sig] = info + c.broadcastedProcessedTxs[info.id] = pTx + return "", nil + } + + // If the transaction regressed from confirmed state, we need to move it back to broadcasted state and rebroadcast. + info.state, pTx.state = Broadcasted, Broadcasted // TODO: may change if we decide to aggregate sigs + delete(c.confirmedTxs, info.id) + c.broadcastedProcessedTxs[info.id] = pTx c.sigToTxInfo[sig] = info return "", nil }) if err != nil { - // If transaction or sig were not found, return + // If transaction or sig were not found return pendingTx{}, err } - // Return the transaction for retrying + // Returns the transaction in case we need to rebroadcast and restart the retry/bumping cycle return pTx, nil } @@ -673,12 +679,12 @@ func newPendingTxContextWithProm(id string) *pendingTxContextWithProm { } } -func (c *pendingTxContextWithProm) New(msg pendingTx, sig solana.Signature, cancel context.CancelFunc) error { - return c.pendingTx.New(msg, sig, cancel) +func (c *pendingTxContextWithProm) New(msg pendingTx) error { + return c.pendingTx.New(msg) } -func (c *pendingTxContextWithProm) AddSignature(id string, sig solana.Signature) error { - return c.pendingTx.AddSignature(id, sig) +func (c *pendingTxContextWithProm) AddSignature(cancel context.CancelFunc, id string, sig solana.Signature) error { + return c.pendingTx.AddSignature(cancel, id, sig) } func (c *pendingTxContextWithProm) OnProcessed(sig solana.Signature) (string, error) { diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 60b208412..9fa3457e7 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -40,13 +40,15 @@ func TestPendingTxContext_add_remove_multiple(t *testing.T) { for i := 0; i < n; i++ { sig, cancel := newProcess() msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) - assert.NoError(t, err) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) + require.NoError(t, err) ids[sig] = msg.id } // cannot add signature for non existent ID - require.Error(t, txs.AddSignature(uuid.New().String(), solana.Signature{})) + require.Error(t, txs.AddSignature(func() {}, uuid.New().String(), solana.Signature{})) // return list of signatures list := txs.ListAll() @@ -75,7 +77,9 @@ func TestPendingTxContext_new(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Check it exists in signature map @@ -112,10 +116,12 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig1) require.NoError(t, err) - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(cancel, msg.id, sig2) require.NoError(t, err) // Check signature map @@ -147,10 +153,12 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) - err = txs.AddSignature(msg.id, sig) + err = txs.AddSignature(cancel, msg.id, sig) require.ErrorIs(t, err, ErrSigAlreadyExists) }) @@ -160,10 +168,12 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig1) require.NoError(t, err) - err = txs.AddSignature("bad id", sig2) + err = txs.AddSignature(cancel, "bad id", sig2) require.ErrorIs(t, err, ErrTransactionNotFound) }) @@ -173,7 +183,9 @@ func TestPendingTxContext_add_signature(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig1) require.NoError(t, err) // Transition to processed state @@ -186,7 +198,7 @@ func TestPendingTxContext_add_signature(t *testing.T) { require.NoError(t, err) require.Equal(t, msg.id, id) - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(cancel, msg.id, sig2) require.ErrorIs(t, err, ErrTransactionNotFound) }) } @@ -202,7 +214,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -238,7 +252,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -261,7 +277,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -289,7 +307,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -307,7 +327,9 @@ func TestPendingTxContext_on_broadcasted_processed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -332,7 +354,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -373,7 +397,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -401,7 +427,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -419,7 +447,9 @@ func TestPendingTxContext_on_confirmed(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to processed state @@ -450,11 +480,13 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig1) require.NoError(t, err) // Add second signature - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(cancel, msg.id, sig2) require.NoError(t, err) // Transition to finalized state @@ -490,11 +522,13 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig1) require.NoError(t, err) // Add second signature - err = txs.AddSignature(msg.id, sig2) + err = txs.AddSignature(cancel, msg.id, sig2) require.NoError(t, err) // Transition to processed state @@ -539,7 +573,9 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig1, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig1) require.NoError(t, err) // Transition to processed state @@ -579,7 +615,9 @@ func TestPendingTxContext_on_finalized(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -604,7 +642,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -637,7 +677,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -675,7 +717,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to fatally errored state @@ -704,7 +748,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -739,7 +785,9 @@ func TestPendingTxContext_on_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to confirmed state @@ -797,7 +845,9 @@ func TestPendingTxContext_on_prebroadcast_error(t *testing.T) { // Create new transaction msg := pendingTx{id: uuid.NewString()} // Add transaction to broadcasted map - err := txs.New(msg, sig, cancel) + err := txs.New(msg) + require.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) require.NoError(t, err) // Transition to errored state @@ -834,14 +884,18 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new broadcasted transaction with extra sig broadcastedMsg := pendingTx{id: uuid.NewString()} - err := txs.New(broadcastedMsg, broadcastedSig1, cancel) + err := txs.New(broadcastedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, broadcastedMsg.id, broadcastedSig1) require.NoError(t, err) - err = txs.AddSignature(broadcastedMsg.id, broadcastedSig2) + err = txs.AddSignature(cancel, broadcastedMsg.id, broadcastedSig2) require.NoError(t, err) // Create new processed transaction processedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(processedMsg, processedSig, cancel) + err = txs.New(processedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, processedMsg.id, processedSig) require.NoError(t, err) id, err := txs.OnProcessed(processedSig) require.NoError(t, err) @@ -849,7 +903,9 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new confirmed transaction confirmedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(confirmedMsg, confirmedSig, cancel) + err = txs.New(confirmedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, confirmedMsg.id, confirmedSig) require.NoError(t, err) id, err = txs.OnConfirmed(confirmedSig) require.NoError(t, err) @@ -857,7 +913,9 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new finalized transaction finalizedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(finalizedMsg, finalizedSig, cancel) + err = txs.New(finalizedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, finalizedMsg.id, finalizedSig) require.NoError(t, err) id, err = txs.OnFinalized(finalizedSig, retentionTimeout) require.NoError(t, err) @@ -865,7 +923,9 @@ func TestPendingTxContext_remove(t *testing.T) { // Create new errored transaction erroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(erroredMsg, erroredSig, cancel) + err = txs.New(erroredMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, erroredMsg.id, erroredSig) require.NoError(t, err) id, err = txs.OnError(erroredSig, retentionTimeout, Errored, 0) require.NoError(t, err) @@ -961,8 +1021,9 @@ func TestPendingTxContext_expired(t *testing.T) { txs := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - err := txs.New(msg, sig, cancel) + err := txs.New(msg) assert.NoError(t, err) + err = txs.AddSignature(cancel, msg.id, sig) msg, exists := txs.broadcastedProcessedTxs[msg.id] require.True(t, exists) @@ -985,15 +1046,16 @@ func TestPendingTxContext_race(t *testing.T) { t.Run("new", func(t *testing.T) { txCtx := newPendingTxContext() var wg sync.WaitGroup + txID := uuid.NewString() wg.Add(2) var err [2]error go func() { - err[0] = txCtx.New(pendingTx{id: uuid.NewString()}, solana.Signature{}, func() {}) + err[0] = txCtx.New(pendingTx{id: txID}) wg.Done() }() go func() { - err[1] = txCtx.New(pendingTx{id: uuid.NewString()}, solana.Signature{}, func() {}) + err[1] = txCtx.New(pendingTx{id: txID}) wg.Done() }() @@ -1004,18 +1066,18 @@ func TestPendingTxContext_race(t *testing.T) { t.Run("add signature", func(t *testing.T) { txCtx := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - createErr := txCtx.New(msg, solana.Signature{}, func() {}) + createErr := txCtx.New(msg) require.NoError(t, createErr) var wg sync.WaitGroup wg.Add(2) var err [2]error go func() { - err[0] = txCtx.AddSignature(msg.id, solana.Signature{1}) + err[0] = txCtx.AddSignature(func() {}, msg.id, solana.Signature{1}) wg.Done() }() go func() { - err[1] = txCtx.AddSignature(msg.id, solana.Signature{1}) + err[1] = txCtx.AddSignature(func() {}, msg.id, solana.Signature{1}) wg.Done() }() @@ -1026,7 +1088,7 @@ func TestPendingTxContext_race(t *testing.T) { t.Run("remove", func(t *testing.T) { txCtx := newPendingTxContext() msg := pendingTx{id: uuid.NewString()} - err := txCtx.New(msg, solana.Signature{}, func() {}) + err := txCtx.New(msg) require.NoError(t, err) var wg sync.WaitGroup wg.Add(2) @@ -1059,13 +1121,17 @@ func TestGetTxState(t *testing.T) { // Create new broadcasted transaction with extra sig broadcastedMsg := pendingTx{id: uuid.NewString()} - err := txs.New(broadcastedMsg, broadcastedSig, cancel) + err := txs.New(broadcastedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, broadcastedMsg.id, broadcastedSig) require.NoError(t, err) var state TxState // Create new processed transaction processedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(processedMsg, processedSig, cancel) + err = txs.New(processedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, processedMsg.id, processedSig) require.NoError(t, err) id, err := txs.OnProcessed(processedSig) require.NoError(t, err) @@ -1077,7 +1143,9 @@ func TestGetTxState(t *testing.T) { // Create new confirmed transaction confirmedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(confirmedMsg, confirmedSig, cancel) + err = txs.New(confirmedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, confirmedMsg.id, confirmedSig) require.NoError(t, err) id, err = txs.OnConfirmed(confirmedSig) require.NoError(t, err) @@ -1089,7 +1157,9 @@ func TestGetTxState(t *testing.T) { // Create new finalized transaction finalizedMsg := pendingTx{id: uuid.NewString()} - err = txs.New(finalizedMsg, finalizedSig, cancel) + err = txs.New(finalizedMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, finalizedMsg.id, finalizedSig) require.NoError(t, err) id, err = txs.OnFinalized(finalizedSig, retentionTimeout) require.NoError(t, err) @@ -1101,7 +1171,9 @@ func TestGetTxState(t *testing.T) { // Create new errored transaction erroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(erroredMsg, erroredSig, cancel) + err = txs.New(erroredMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, erroredMsg.id, erroredSig) require.NoError(t, err) id, err = txs.OnError(erroredSig, retentionTimeout, Errored, 0) require.NoError(t, err) @@ -1113,7 +1185,9 @@ func TestGetTxState(t *testing.T) { // Create new fatally errored transaction fatallyErroredMsg := pendingTx{id: uuid.NewString()} - err = txs.New(fatallyErroredMsg, fatallyErroredSig, cancel) + err = txs.New(fatallyErroredMsg) + require.NoError(t, err) + err = txs.AddSignature(cancel, fatallyErroredMsg.id, fatallyErroredSig) require.NoError(t, err) id, err = txs.OnError(fatallyErroredSig, retentionTimeout, FatallyErrored, 0) require.NoError(t, err) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index b0e63f513..e66cb0354 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -216,10 +216,16 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("tx failed initial transmit: %w", errors.Join(initSendErr, stateTransitionErr)) } - // Store tx signature and cancel function - if err := txm.txs.New(msg, sig, cancel); err != nil { - cancel() // Cancel context when exiting early - return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to save tx signature (%s) to inflight txs: %w", sig, err) + // Create new transaction in memory + if err := txm.txs.New(msg); err != nil { + cancel() + return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to create new transaction: %w", err) + } + + // Associate initial signature and cancel func to tx + if err := txm.txs.AddSignature(cancel, msg.id, sig); err != nil { + cancel() + return solanaGo.Transaction{}, "", solanaGo.Signature{}, fmt.Errorf("failed to save initial signature (%s) to inflight txs: %w", sig, err) } txm.lggr.Debugw("tx initial broadcast", "id", msg.id, "fee", msg.cfg.BaseComputeUnitPrice, "signature", sig) @@ -228,7 +234,7 @@ func (txm *Txm) sendWithRetry(ctx context.Context, msg pendingTx) (solanaGo.Tran txm.done.Add(1) go func() { defer txm.done.Done() - txm.retryTx(ctx, msg, initTx, sig) + txm.retryTx(ctx, cancel, msg, initTx, sig) }() // Return signed tx, id, signature for use in simulation @@ -279,11 +285,12 @@ func (txm *Txm) buildTx(ctx context.Context, msg pendingTx, retryCount int) (sol // retryTx contains the logic for retrying the transaction, including exponential backoff and fee bumping. // Retries until context cancelled by timeout or called externally. // It uses handleRetry helper function to handle each retry attempt. -func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature) { +func (txm *Txm) retryTx(ctx context.Context, cancel context.CancelFunc, msg pendingTx, currentTx solanaGo.Transaction, sig solanaGo.Signature) { // Initialize signature list with initialTx signature. This list will be used to add new signatures and track retry attempts. sigs := &signatureList{} sigs.Allocate() if initSetErr := sigs.Set(0, sig); initSetErr != nil { + cancel() txm.lggr.Errorw("failed to save initial signature in signature list", "error", initSetErr) return } @@ -328,7 +335,7 @@ func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.T wg.Add(1) go func(bump bool, count int, retryTx solanaGo.Transaction) { defer wg.Done() - txm.handleRetry(ctx, msg, bump, count, retryTx, sigs) + txm.handleRetry(ctx, cancel, msg, bump, count, retryTx, sigs) }(shouldBump, bumpCount, currentTx) } @@ -342,7 +349,7 @@ func (txm *Txm) retryTx(ctx context.Context, msg pendingTx, currentTx solanaGo.T } // handleRetry handles the logic for each retry attempt, including sending the transaction, updating signatures, and logging. -func (txm *Txm) handleRetry(ctx context.Context, msg pendingTx, bump bool, count int, retryTx solanaGo.Transaction, sigs *signatureList) { +func (txm *Txm) handleRetry(ctx context.Context, cancel context.CancelFunc, msg pendingTx, bump bool, count int, retryTx solanaGo.Transaction, sigs *signatureList) { // send retry transaction retrySig, err := txm.sendTx(ctx, &retryTx) if err != nil { @@ -357,7 +364,7 @@ func (txm *Txm) handleRetry(ctx context.Context, msg pendingTx, bump bool, count // if bump is true, update signature list and set new signature in space already allocated. if bump { - if err := txm.txs.AddSignature(msg.id, retrySig); err != nil { + if err := txm.txs.AddSignature(cancel, msg.id, retrySig); err != nil { txm.lggr.Warnw("error in adding retry transaction", "error", err, "id", msg.id) return } @@ -538,20 +545,26 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status // Check if tx has been reorged by detecting if we had a status regression // If so, we'll handle the reorg by updating the status in our in-memory layer and retrying the transaction for that sig. currentTxState := convertStatus(status) - if isStatusRegression(txInfo.state, currentTxState) { + if regressionType, isRegressed := isStatusRegression(txInfo.state, currentTxState); isRegressed { txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) - // Handle reorg in our in memory layer and retry transaction pTx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) return err } - retryCtx, _ := context.WithTimeout(ctx, pTx.cfg.Timeout) // TODO: Ask here. How should we handle the ctx? - txm.done.Add(1) - go func() { - defer txm.done.Done() - txm.retryTx(retryCtx, pTx, pTx.tx, sig) - }() + + // If the transaction is regressing from a confirmed state, we will restart the retry/bumping cycle + if regressionType == FromConfirmed { + retryCtx, cancel := context.WithTimeout(ctx, pTx.cfg.Timeout) + txm.done.Add(1) + go func() { + defer txm.done.Done() + txm.retryTx(retryCtx, cancel, pTx, pTx.tx, sig) + }() + } + // If the transaction is regressing from a processed state, the retry/bumping cycle is still running for the original tx. + // We will not restart the retry/bumping cycle for this transaction right now. + // If it expires and TxExpiredRebroadcast is enabled, it will be rebroadcasted as per the expiration rebroadcast logic. } return nil diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 4f268aed2..59f5d461c 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1503,8 +1503,6 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { return sig1, nil } - // Mock LatestBlockhash to return an invalid blockhash less than slotHeight - // We won't use it as there will be no rebroadcasts txes to process. All txes will be confirmed before. slotHeightFunc := func() (uint64, error) { return uint64(1500), nil } @@ -1514,7 +1512,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { defer func() { callCount++ }() return &rpc.GetLatestBlockhashResult{ Value: &rpc.LatestBlockhashResult{ - LastValidBlockHeight: uint64(1000), + LastValidBlockHeight: uint64(2000), }, }, nil } diff --git a/pkg/solana/txm/utils.go b/pkg/solana/txm/utils.go index 067e16bbd..26591bade 100644 --- a/pkg/solana/txm/utils.go +++ b/pkg/solana/txm/utils.go @@ -51,6 +51,35 @@ func (s TxState) String() string { } } +type regressionType int + +const ( + FromConfirmed regressionType = iota + FromProcessed +) + +// isStatusRegression checks if the current status is a regression compared to the previous status: +// - Confirmed -> Processed, Broadcasted: should not regress +// - Processed -> Broadcasted: should not regress +// Returns true if a regression is detected, indicating a possible re-org. +func isStatusRegression(previous, current TxState) (regressionType, bool) { + switch previous { + case Confirmed: + // Confirmed transactions should not regress to Processed or Broadcasted. + if current != Confirmed && current != Finalized && current != Errored { + return FromConfirmed, true + } + case Processed: + // Processed transactions should not regress to Broadcasted. + if current != Processed && current != Confirmed && current != Finalized && current != Errored { + return FromProcessed, true + } + default: + return 0, false + } + return 0, false +} + type statuses struct { sigs []solana.Signature res []*rpc.SignatureStatusesResult @@ -111,28 +140,6 @@ func convertStatus(res *rpc.SignatureStatusesResult) TxState { return NotFound } -// isStatusRegression checks if the current status is a regression compared to the previous status: -// - Confirmed -> Processed, Broadcasted: should not regress -// - Processed -> Broadcasted: should not regress -// Returns true if a regression is detected, indicating a possible re-org. -func isStatusRegression(previous, current TxState) bool { - switch previous { - case Confirmed: - // Confirmed transactions should not regress to Processed or Broadcasted. - if current != Confirmed && current != Finalized && current != Errored { - return true - } - case Processed: - // Processed transactions should not regress to Broadcasted. - if current != Processed && current != Confirmed && current != Finalized && current != Errored { - return true - } - default: - return false - } - return false -} - type signatureList struct { sigs []solana.Signature lock sync.RWMutex From cef6a91c8e19f8a28c266619c19cf30cb19f1ec5 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 18:50:16 -0300 Subject: [PATCH 20/31] fix ci --- integration-tests/go.mod | 4 ++-- integration-tests/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 888056bfb..c38aa221d 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -15,12 +15,12 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 github.com/rs/zerolog v1.33.0 github.com/smartcontractkit/chainlink-common v0.3.1-0.20241127162636-07aa781ee1f4 - github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241127190942-9a418b680971 + github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241127201057-3c9282e39749 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.17 github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.9 github.com/smartcontractkit/chainlink/deployment v0.0.0-20241127192805-54ea74a13bfe github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241127192805-54ea74a13bfe - github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241127200605-786894ba3036 + github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241127212059-7a8a07985766 github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.34.0 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index e8ce2ca52..857f4d5bb 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1410,8 +1410,8 @@ github.com/smartcontractkit/chainlink/deployment v0.0.0-20241127192805-54ea74a13 github.com/smartcontractkit/chainlink/deployment v0.0.0-20241127192805-54ea74a13bfe/go.mod h1:ueUOL11tGBu1TTonZcIeD6/3av2iZE5AydxtclG8Dvo= github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241127192805-54ea74a13bfe h1:93zlj92vuyjALbjOJ5NahdFQjAwEVplXQIzzqrvFMNc= github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20241127192805-54ea74a13bfe/go.mod h1:MinN1uhp3ygTM9ctNwJgLx7LyBPmwAR1NK18HBK8OEU= -github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241127200605-786894ba3036 h1:wUxtbcmVWldfGijbu3H8U2oNlmgtkqE/dwtsxIYBnAs= -github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241127200605-786894ba3036/go.mod h1:pTLM5KDUNXZxRoE0hQCyInFGHZ/tcPNefxX2j8XntCE= +github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241127212059-7a8a07985766 h1:IZ9/RvaF94wyLk16IyJdiLJ2lqly8lhKly7XdMUjtDk= +github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241127212059-7a8a07985766/go.mod h1:54UOlBv6wcvapka94RTgaF82uXkPAwwhpsDcmm5KsBc= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7/go.mod h1:FX7/bVdoep147QQhsOPkYsPEXhGZjeYx6lBSaSXtZOA= github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12 h1:NzZGjaqez21I3DU7objl3xExTH4fxYvzTqar8DC6360= From 3b3a71bb5be61204786fa778b16b66fcba26ce86 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 3 Dec 2024 19:08:17 -0300 Subject: [PATCH 21/31] fix lint --- pkg/solana/txm/pendingtx.go | 8 +++++--- pkg/solana/txm/pendingtx_test.go | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index b39b6b0dc..33fd54677 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -605,11 +605,13 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { if !exists { return "", ErrSigDoesNotExist } - var broadcastedProcessedExists, confirmedExists bool - if tx, broadcastedProcessedExists := c.broadcastedProcessedTxs[info.id]; broadcastedProcessedExists { + + tx, broadcastedProcessedExists := c.broadcastedProcessedTxs[info.id] + if broadcastedProcessedExists { pTx = tx } - if tx, confirmedExists := c.confirmedTxs[info.id]; confirmedExists { + tx, confirmedExists := c.confirmedTxs[info.id] + if confirmedExists { pTx = tx } diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 9fa3457e7..b10fe0b7b 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -41,9 +41,9 @@ func TestPendingTxContext_add_remove_multiple(t *testing.T) { sig, cancel := newProcess() msg := pendingTx{id: uuid.NewString()} err := txs.New(msg) - require.NoError(t, err) + assert.NoError(t, err) err = txs.AddSignature(cancel, msg.id, sig) - require.NoError(t, err) + assert.NoError(t, err) ids[sig] = msg.id } @@ -1024,6 +1024,7 @@ func TestPendingTxContext_expired(t *testing.T) { err := txs.New(msg) assert.NoError(t, err) err = txs.AddSignature(cancel, msg.id, sig) + assert.NoError(t, err) msg, exists := txs.broadcastedProcessedTxs[msg.id] require.True(t, exists) From ef782c31924213e6d97eda091d402a2f2dbd5ca1 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 4 Dec 2024 11:38:29 -0300 Subject: [PATCH 22/31] handle multiple sigs case --- pkg/solana/txm/pendingtx.go | 53 +++++++++++++++++++++++++++++++++++++ pkg/solana/txm/txm.go | 16 ++++++++--- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 33fd54677..f032faf60 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -50,6 +50,10 @@ type PendingTxContext interface { TrimFinalizedErroredTxs() int // GetSignatureInfo returns the transaction ID and TxState for the provided signature GetSignatureInfo(sig solana.Signature) (txInfo, error) + // TxHasReorg determines whether a reorg has occurred for a given tx. + // It achieves this by comparing the highest aggregated state across all associated signatures with the current state of the transaction. + // If the highest aggregated state is less than the current state, a reorg has occurred and we need to handle it. + TxHasReorg(id string) bool // OnReorg resets the transaction state to Broadcasted for the given signature and returns the pendingTx for retrying. OnReorg(sig solana.Signature) (pendingTx, error) } @@ -644,6 +648,51 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { return pTx, nil } +// TxHasReorg determines whether a reorg has occurred for a given tx. +// It achieves this by comparing the highest aggregated state across all associated signatures with the current state of the transaction. +// If the highest aggregated state is less than the current state, a reorg has occurred and we need to handle it. +func (c *pendingTxContext) TxHasReorg(id string) bool { + var pTx pendingTx + var broadcastedExists, confirmedExists bool + statePriority := map[TxState]int{ + Broadcasted: 1, + Processed: 2, + Confirmed: 3, + } + highestSigAggState := Broadcasted + + c.lock.RLock() + defer c.lock.RUnlock() + // Check if the transaction is still in a non finalized/errored state + tx, broadcastedExists := c.broadcastedProcessedTxs[id] + if broadcastedExists { + pTx = tx + } + tx, confirmedExists = c.confirmedTxs[id] + if confirmedExists { + pTx = tx + } + if !broadcastedExists && !confirmedExists { + return false + } + + // Get the highest state among all signatures + for _, sig := range pTx.signatures { + info, exists := c.sigToTxInfo[sig] + if !exists { + continue + } + if priority, ok := statePriority[info.state]; ok { + if priority > statePriority[highestSigAggState] { + highestSigAggState = info.state + } + } + } + + // If the highest state among all signatures is less than the transaction state, then a reorg has occurred + return statePriority[highestSigAggState] < statePriority[pTx.state] +} + func (c *pendingTxContext) withReadLock(fn func() error) error { c.lock.RLock() defer c.lock.RUnlock() @@ -778,3 +827,7 @@ func (c *pendingTxContextWithProm) GetSignatureInfo(sig solana.Signature) (txInf func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, error) { return c.pendingTx.OnReorg(sig) } + +func (c *pendingTxContextWithProm) TxHasReorg(id string) bool { + return c.pendingTx.TxHasReorg(id) +} diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index e66cb0354..f0c34c81a 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -542,11 +542,20 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status return err } - // Check if tx has been reorged by detecting if we had a status regression - // If so, we'll handle the reorg by updating the status in our in-memory layer and retrying the transaction for that sig. + // Check if the sig has been reorged by detecting if it had a status regression + // Confirmed -> Processed || Not Found + // Processed -> Not Found currentTxState := convertStatus(status) if regressionType, isRegressed := isStatusRegression(txInfo.state, currentTxState); isRegressed { - txm.lggr.Warnw("potential re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) + // Check if the signature reorg causes a reorg in the tx state + // If the tx is not in a re-org state, we will not handle it despite the sig being reorged. + // As we have multiple sigs inflight for a single tx, having a reorg for one sig does not mean all the sigs were reorged. + // The tx may still be on this sig "previous state" if other sigs were not reorged. + if !txm.txs.TxHasReorg(txInfo.id) { + return nil + } + + txm.lggr.Warnw("re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) pTx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) @@ -566,7 +575,6 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status // We will not restart the retry/bumping cycle for this transaction right now. // If it expires and TxExpiredRebroadcast is enabled, it will be rebroadcasted as per the expiration rebroadcast logic. } - return nil } From 08b0c6e822c65fa51e628e9280f77290cc37ff47 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 4 Dec 2024 12:38:22 -0300 Subject: [PATCH 23/31] improve comment --- pkg/solana/txm/txm.go | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index f0c34c81a..045a94f18 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -532,37 +532,47 @@ func (txm *Txm) handleErrorSignatureStatus(sig solanaGo.Signature, status *rpc.S } } -// handleReorg handles the case where a transaction signature is in a potential reorg state on-chain. -// It updates the transaction state in the local memory and restarts the retry/bumping cycle for the transaction associated to that sig. +// handleReorg detects and manages transaction state regressions (re-orgs) for a given signature. +// +// A re-org occurs when the blockchain state of a signature regresses to: +// - Confirmed -> Processed || Not Found +// - Processed -> Not Found +// +// This function determines if the signature’s state regression impacts the overall transaction state and, if so, takes appropriate action: +// - For regressions from "Confirmed", our in memory layer is updated, the tx is rebroadcasted, and the retry/bumping cycle is restarted. +// - For regressions from "Processed", the existing retry/bumping cycle is still running, so no immediate action is needed. We only update our in-memory state to Broadcasted. +// Future rebroadcasts, will be handled by the TxExpirationRebroadcast logic (if enabled) when the transaction expires. func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status *rpc.SignatureStatusesResult) error { - // Retrieve last seen status for the tx associated to this sig in our in-memory layer. + // Retrieve the last known status of the transaction associated with this signature from the in-memory layer. txInfo, err := txm.txs.GetSignatureInfo(sig) if err != nil { txm.lggr.Errorw("failed to get signature info when checking for potential re-orgs", "signature", sig, "error", err) return err } - // Check if the sig has been reorged by detecting if it had a status regression - // Confirmed -> Processed || Not Found - // Processed -> Not Found + // Check if the sig status has regressed to indicate a re-org. + // A regression is identified when the state transitions as follows: + // - Confirmed -> Processed || Not Found + // - Processed -> Not Found currentTxState := convertStatus(status) if regressionType, isRegressed := isStatusRegression(txInfo.state, currentTxState); isRegressed { - // Check if the signature reorg causes a reorg in the tx state - // If the tx is not in a re-org state, we will not handle it despite the sig being reorged. - // As we have multiple sigs inflight for a single tx, having a reorg for one sig does not mean all the sigs were reorged. - // The tx may still be on this sig "previous state" if other sigs were not reorged. + // Determine if the sig regression affects the transaction state. + // If the tx isn't considered re-orged, skip further processing. + // Multiple signatures may be in-flight for a single transaction, so a re-org + // for one signature doesn't necessarily mean the transaction state has regressed. if !txm.txs.TxHasReorg(txInfo.id) { return nil } txm.lggr.Warnw("re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) + // Handle re-org for the transaction and update the in-memory state. pTx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) return err } - // If the transaction is regressing from a confirmed state, we will restart the retry/bumping cycle + // For regressions from "Confirmed", restart the retry/bumping cycle. if regressionType == FromConfirmed { retryCtx, cancel := context.WithTimeout(ctx, pTx.cfg.Timeout) txm.done.Add(1) @@ -571,9 +581,10 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status txm.retryTx(retryCtx, cancel, pTx, pTx.tx, sig) }() } - // If the transaction is regressing from a processed state, the retry/bumping cycle is still running for the original tx. - // We will not restart the retry/bumping cycle for this transaction right now. - // If it expires and TxExpiredRebroadcast is enabled, it will be rebroadcasted as per the expiration rebroadcast logic. + // For regressions from "Processed" do not restart the cycle immediately. + // The retry/bumping cycle for the original transaction is still active. + // If rebroadcasting becomes necessary later, it will be handled via the + // TxExpirationRebroadcast logic (if enabled) when the transaction expires. } return nil } From 63a5f3f4957b3cee9c8527f814319d8adbbc6cf9 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Wed, 4 Dec 2024 12:54:44 -0300 Subject: [PATCH 24/31] improve logic and comments --- pkg/solana/txm/pendingtx.go | 19 ++++++++----------- pkg/solana/txm/txm.go | 6 +++--- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index f032faf60..537f40c3c 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -624,19 +624,16 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { return "", ErrTransactionNotFound } - // If the transaction regressed from processed state, we only need to reset the state - if broadcastedProcessedExists { - info.state, pTx.state = Broadcasted, Broadcasted - c.sigToTxInfo[sig] = info - c.broadcastedProcessedTxs[info.id] = pTx - return "", nil + // reset state to broadcasted and update the transaction in the broadcasted map + info.state, pTx.state = Broadcasted, Broadcasted + c.sigToTxInfo[sig] = info + c.broadcastedProcessedTxs[info.id] = pTx + + // If the transaction regressed from confirmed state, we also need to remove it from the confirmed map + if confirmedExists { + delete(c.confirmedTxs, info.id) } - // If the transaction regressed from confirmed state, we need to move it back to broadcasted state and rebroadcast. - info.state, pTx.state = Broadcasted, Broadcasted // TODO: may change if we decide to aggregate sigs - delete(c.confirmedTxs, info.id) - c.broadcastedProcessedTxs[info.id] = pTx - c.sigToTxInfo[sig] = info return "", nil }) if err != nil { diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 045a94f18..aa387821f 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -565,14 +565,14 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status } txm.lggr.Warnw("re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) - // Handle re-org for the transaction and update the in-memory state. + // update the in-memory state and return the transaction associated with the signature for rebroadcasting and restarting retry/bump cycle if needed pTx, err := txm.txs.OnReorg(sig) if err != nil { - txm.lggr.Errorw("failed to handle potential re-org", "signature", sig, "id", pTx.id, "error", err) + txm.lggr.Errorw("failed to handle re-org", "signature", sig, "id", pTx.id, "error", err) return err } - // For regressions from "Confirmed", restart the retry/bumping cycle. + // For regressions from "Confirmed", rebroadcast tx and restart retry/bumping cycle. if regressionType == FromConfirmed { retryCtx, cancel := context.WithTimeout(ctx, pTx.cfg.Timeout) txm.done.Add(1) From 23f42d1eb3707690858bb88c901f71c7d3f3e322 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Fri, 6 Dec 2024 18:20:01 -0300 Subject: [PATCH 25/31] address feedback --- pkg/solana/txm/pendingtx.go | 13 ++----------- pkg/solana/txm/txm.go | 19 ++++++++----------- pkg/solana/txm/utils.go | 4 ++-- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 537f40c3c..69c52eaad 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -651,11 +651,6 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { func (c *pendingTxContext) TxHasReorg(id string) bool { var pTx pendingTx var broadcastedExists, confirmedExists bool - statePriority := map[TxState]int{ - Broadcasted: 1, - Processed: 2, - Confirmed: 3, - } highestSigAggState := Broadcasted c.lock.RLock() @@ -679,15 +674,11 @@ func (c *pendingTxContext) TxHasReorg(id string) bool { if !exists { continue } - if priority, ok := statePriority[info.state]; ok { - if priority > statePriority[highestSigAggState] { - highestSigAggState = info.state - } - } + highestSigAggState = max(highestSigAggState, info.state) } // If the highest state among all signatures is less than the transaction state, then a reorg has occurred - return statePriority[highestSigAggState] < statePriority[pTx.state] + return highestSigAggState < pTx.state } func (c *pendingTxContext) withReadLock(fn func() error) error { diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index aa387821f..7a3dc94ef 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -462,6 +462,8 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr // sig not found could mean invalid tx or not picked up yet, keep polling if status == nil { txm.handleNotFoundSignatureStatus(sig) + // check if a potential re-org has occurred for this sig and handle it + txm.handleReorg(ctx, sig, status) continue } @@ -474,7 +476,9 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr switch status.ConfirmationStatus { case rpc.ConfirmationStatusProcessed: // if signature is processed, keep polling for confirmed or finalized status + // we also need to check if a potential re-org has occurred for this sig and handle it txm.handleProcessedSignatureStatus(sig) + txm.handleReorg(ctx, sig, status) case rpc.ConfirmationStatusConfirmed: // if signature is confirmed, keep polling for finalized status txm.handleConfirmedSignatureStatus(sig) @@ -484,12 +488,6 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr default: txm.lggr.Warnw("unknown confirmation status", "signature", sig, "status", status.ConfirmationStatus) } - - // check if a potential re-org has occurred for this sig and handle it - err := txm.handleReorg(ctx, sig, status) - if err != nil { - continue - } } }(i) } @@ -542,12 +540,12 @@ func (txm *Txm) handleErrorSignatureStatus(sig solanaGo.Signature, status *rpc.S // - For regressions from "Confirmed", our in memory layer is updated, the tx is rebroadcasted, and the retry/bumping cycle is restarted. // - For regressions from "Processed", the existing retry/bumping cycle is still running, so no immediate action is needed. We only update our in-memory state to Broadcasted. // Future rebroadcasts, will be handled by the TxExpirationRebroadcast logic (if enabled) when the transaction expires. -func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status *rpc.SignatureStatusesResult) error { +func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status *rpc.SignatureStatusesResult) { // Retrieve the last known status of the transaction associated with this signature from the in-memory layer. txInfo, err := txm.txs.GetSignatureInfo(sig) if err != nil { txm.lggr.Errorw("failed to get signature info when checking for potential re-orgs", "signature", sig, "error", err) - return err + return } // Check if the sig status has regressed to indicate a re-org. @@ -561,7 +559,7 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status // Multiple signatures may be in-flight for a single transaction, so a re-org // for one signature doesn't necessarily mean the transaction state has regressed. if !txm.txs.TxHasReorg(txInfo.id) { - return nil + return } txm.lggr.Warnw("re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) @@ -569,7 +567,7 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status pTx, err := txm.txs.OnReorg(sig) if err != nil { txm.lggr.Errorw("failed to handle re-org", "signature", sig, "id", pTx.id, "error", err) - return err + return } // For regressions from "Confirmed", rebroadcast tx and restart retry/bumping cycle. @@ -586,7 +584,6 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status // If rebroadcasting becomes necessary later, it will be handled via the // TxExpirationRebroadcast logic (if enabled) when the transaction expires. } - return nil } // handleProcessedSignatureStatus handles the case where a transaction signature is in the "processed" state on-chain. diff --git a/pkg/solana/txm/utils.go b/pkg/solana/txm/utils.go index 26591bade..3373bd60c 100644 --- a/pkg/solana/txm/utils.go +++ b/pkg/solana/txm/utils.go @@ -66,12 +66,12 @@ func isStatusRegression(previous, current TxState) (regressionType, bool) { switch previous { case Confirmed: // Confirmed transactions should not regress to Processed or Broadcasted. - if current != Confirmed && current != Finalized && current != Errored { + if current == Processed || current == Broadcasted { return FromConfirmed, true } case Processed: // Processed transactions should not regress to Broadcasted. - if current != Processed && current != Confirmed && current != Finalized && current != Errored { + if current == Broadcasted { return FromProcessed, true } default: From 53948e1f97a33fe968cd44ea5302d76195dbedf1 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Mon, 9 Dec 2024 13:03:07 -0300 Subject: [PATCH 26/31] add comment --- pkg/solana/txm/pendingtx.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index b58d69115..fb6b8e2ce 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -615,7 +615,13 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { return "", ErrTransactionNotFound } - // reset state to broadcasted and update the transaction in the broadcasted map + // Reset the transaction state to 'Broadcasted' upon detecting a reorg. + // Even if the transaction might have already progressed to 'Processed' before the reorg, + // we reset it to 'Broadcasted' for simplicity. + // Any state advancements (e.g., moving to 'Processed' or 'Confirmed') will be picked up + // on the next status polling cycle. + // This approach does not introduce any risk with the expiration logic since + // we check for status changes before considering a transaction for expiration. info.state, pTx.state = Broadcasted, Broadcasted c.sigToTxInfo[sig] = info c.broadcastedProcessedTxs[info.id] = pTx From fdc80689a2d070c7b9efd890bfc8000d9d4da46d Mon Sep 17 00:00:00 2001 From: Farber98 Date: Mon, 9 Dec 2024 19:42:25 -0300 Subject: [PATCH 27/31] tests and fix some bugs --- pkg/solana/txm/pendingtx.go | 82 +++++--- pkg/solana/txm/pendingtx_test.go | 217 ++++++++++++++++++++ pkg/solana/txm/txm.go | 12 +- pkg/solana/txm/txm_internal_test.go | 304 ++++++++++++++++++++++++++++ pkg/solana/txm/utils.go | 10 +- 5 files changed, 589 insertions(+), 36 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index fb6b8e2ce..3ef1b1f3e 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -50,12 +50,14 @@ type PendingTxContext interface { TrimFinalizedErroredTxs() int // GetSignatureInfo returns the transaction ID and TxState for the provided signature GetSignatureInfo(sig solana.Signature) (txInfo, error) + // UpdateSignatureStatus updates the status of a signature in the SigToTxInfo map + UpdateSignatureStatus(sig solana.Signature, status TxState) error // TxHasReorg determines whether a reorg has occurred for a given tx. // It achieves this by comparing the highest aggregated state across all associated signatures with the current state of the transaction. // If the highest aggregated state is less than the current state, a reorg has occurred and we need to handle it. TxHasReorg(id string) bool // OnReorg resets the transaction state to Broadcasted for the given signature and returns the pendingTx for retrying. - OnReorg(sig solana.Signature) (pendingTx, error) + OnReorg(sig solana.Signature, id string) (pendingTx, error) } // finishedTx is used to store info required to track transactions to finality or error @@ -569,19 +571,12 @@ func (c *pendingTxContext) GetSignatureInfo(sig solana.Signature) (txInfo, error return info, nil } -func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { - // Acquire a read lock to check if the signature exists and needs to be reset +func (c *pendingTxContext) OnReorg(sig solana.Signature, id string) (pendingTx, error) { err := c.withReadLock(func() error { - // Check if the signature is still being tracked - info, exists := c.sigToTxInfo[sig] - if !exists { - return ErrSigDoesNotExist - } - // Check if the transaction is still in a non finalized/errored state var broadcastedExists, confirmedExists bool - _, broadcastedExists = c.broadcastedProcessedTxs[info.id] - _, confirmedExists = c.confirmedTxs[info.id] + _, broadcastedExists = c.broadcastedProcessedTxs[id] + _, confirmedExists = c.confirmedTxs[id] if !broadcastedExists && !confirmedExists { return ErrTransactionNotFound } @@ -593,19 +588,14 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { } var pTx pendingTx - // Acquire a write lock to perform the state reset + // Acquire a write lock to perform the state reset if needed _, err = c.withWriteLock(func() (string, error) { - // Retrieve sig and tx again inside the write lock - info, exists := c.sigToTxInfo[sig] - if !exists { - return "", ErrSigDoesNotExist - } - - tx, broadcastedProcessedExists := c.broadcastedProcessedTxs[info.id] + // Retrieve tx again inside the write lock + tx, broadcastedProcessedExists := c.broadcastedProcessedTxs[id] if broadcastedProcessedExists { pTx = tx } - tx, confirmedExists := c.confirmedTxs[info.id] + tx, confirmedExists := c.confirmedTxs[id] if confirmedExists { pTx = tx } @@ -622,19 +612,18 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature) (pendingTx, error) { // on the next status polling cycle. // This approach does not introduce any risk with the expiration logic since // we check for status changes before considering a transaction for expiration. - info.state, pTx.state = Broadcasted, Broadcasted - c.sigToTxInfo[sig] = info - c.broadcastedProcessedTxs[info.id] = pTx + pTx.state = Broadcasted + c.broadcastedProcessedTxs[id] = pTx // If the transaction regressed from confirmed state, we also need to remove it from the confirmed map if confirmedExists { - delete(c.confirmedTxs, info.id) + delete(c.confirmedTxs, id) } return "", nil }) if err != nil { - // If transaction or sig were not found + // If transaction was not found return pendingTx{}, err } @@ -678,6 +667,41 @@ func (c *pendingTxContext) TxHasReorg(id string) bool { return highestSigAggState < pTx.state } +func (c *pendingTxContext) UpdateSignatureStatus(sig solana.Signature, status TxState) error { + // Acquire a read lock to check if the signature exists and needs to be reset + err := c.withReadLock(func() error { + // Check if the signature is still being tracked + _, exists := c.sigToTxInfo[sig] + if !exists { + return ErrSigDoesNotExist + } + return nil + }) + if err != nil { + // If sig not found, return + return err + } + + // Acquire a write lock to perform the state reset + _, err = c.withWriteLock(func() (string, error) { + // Retrieve sig again inside the write lock + info, exists := c.sigToTxInfo[sig] + if !exists { + return "", ErrSigDoesNotExist + } + // Update the status of the signature + info.state = status + c.sigToTxInfo[sig] = info + return "", nil + }) + if err != nil { + // If sig was not found + return err + } + + return nil +} + func (c *pendingTxContext) withReadLock(fn func() error) error { c.lock.RLock() defer c.lock.RUnlock() @@ -809,10 +833,14 @@ func (c *pendingTxContextWithProm) GetSignatureInfo(sig solana.Signature) (txInf return c.pendingTx.GetSignatureInfo(sig) } -func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature) (pendingTx, error) { - return c.pendingTx.OnReorg(sig) +func (c *pendingTxContextWithProm) OnReorg(sig solana.Signature, id string) (pendingTx, error) { + return c.pendingTx.OnReorg(sig, id) } func (c *pendingTxContextWithProm) TxHasReorg(id string) bool { return c.pendingTx.TxHasReorg(id) } + +func (c *pendingTxContextWithProm) UpdateSignatureStatus(sig solana.Signature, status TxState) error { + return c.pendingTx.UpdateSignatureStatus(sig, status) +} diff --git a/pkg/solana/txm/pendingtx_test.go b/pkg/solana/txm/pendingtx_test.go index 2400ea936..25dab6bc8 100644 --- a/pkg/solana/txm/pendingtx_test.go +++ b/pkg/solana/txm/pendingtx_test.go @@ -1375,3 +1375,220 @@ func TestPendingTxContext_ListAllExpiredBroadcastedTxs(t *testing.T) { }) } } + +func TestPendingTxContext_UpdateSignatureStatus(t *testing.T) { + t.Parallel() + txs := newPendingTxContext() + sig := randomSignature(t) + txID := uuid.NewString() + cancelFunc := func() {} + + // Add new transaction and signature + tx := pendingTx{id: txID} + require.NoError(t, txs.New(tx)) + require.NoError(t, txs.AddSignature(cancelFunc, txID, sig)) + + // updates signature status successfully + err := txs.UpdateSignatureStatus(sig, Confirmed) + require.NoError(t, err) + txInfo, exists := txs.sigToTxInfo[sig] + require.True(t, exists) + require.Equal(t, Confirmed, txInfo.state) + + // updating non-existent signature returs error + nonExistentSig := randomSignature(t) + err = txs.UpdateSignatureStatus(nonExistentSig, Confirmed) + require.ErrorIs(t, err, ErrSigDoesNotExist) + + // Test concurrent updates to ensure thread safety + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func(status TxState) { + defer wg.Done() + err := txs.UpdateSignatureStatus(sig, status) + require.NoError(t, err) + }(Confirmed) + } + wg.Wait() + + // Verify final status + txInfo, exists = txs.sigToTxInfo[sig] + require.True(t, exists) + require.Equal(t, Confirmed, txInfo.state) +} + +func createTxAndAddSig(t *testing.T, txs *pendingTxContext) (string, solana.Signature) { + sig := randomSignature(t) + txID := uuid.NewString() + tx := pendingTx{id: txID} + require.NoError(t, txs.New(tx)) + require.NoError(t, txs.AddSignature(func() {}, txID, sig)) + return txID, sig +} + +func TestPendingTxContext_OnReorg(t *testing.T) { + t.Parallel() + txs := newPendingTxContext() + t.Run("successfully reset transaction from Processed to Broadcasted", func(t *testing.T) { + // Transition to Processed state + txID, sig := createTxAndAddSig(t, txs) + _, err := txs.OnProcessed(sig) + require.NoError(t, err) + + // Call OnReorg + pTx, err := txs.OnReorg(sig, txID) + require.NoError(t, err) + require.Equal(t, Broadcasted, pTx.state) + + // Verify the transaction's state is reset to Broadcasted + txInfo, exists := txs.broadcastedProcessedTxs[txID] + require.True(t, exists) + require.Equal(t, Broadcasted, txInfo.state) + }) + + t.Run("successfully reset transaction from Confirmed to Broadcasted", func(t *testing.T) { + // Transition to Processed and then Confirmed state + txID, sig := createTxAndAddSig(t, txs) + _, err := txs.OnProcessed(sig) + require.NoError(t, err) + _, err = txs.OnConfirmed(sig) + require.NoError(t, err) + + // Call OnReorg + pTx, err := txs.OnReorg(sig, txID) + require.NoError(t, err) + require.Equal(t, Broadcasted, pTx.state) + + // Verify the transaction's state is reset to Broadcasted + txInfo, exists := txs.broadcastedProcessedTxs[txID] + require.True(t, exists) + require.Equal(t, Broadcasted, txInfo.state) + + // Ensure it's removed from confirmed transactions + _, exists = txs.confirmedTxs[txID] + require.False(t, exists) + }) + + t.Run("fail to reset transaction in Finalized state", func(t *testing.T) { + // Transition to Processed, Confirmed, and then Finalized state + txID, sig := createTxAndAddSig(t, txs) + _, err := txs.OnProcessed(sig) + require.NoError(t, err) + _, err = txs.OnConfirmed(sig) + require.NoError(t, err) + _, err = txs.OnFinalized(sig, 10*time.Second) + require.NoError(t, err) + + // Call OnReorg + _, err = txs.OnReorg(sig, txID) + require.Error(t, err) + require.Equal(t, ErrTransactionNotFound, err) + }) + + t.Run("fail to reset transaction in Errored state", func(t *testing.T) { + // Transition to Errored state + txID, sig := createTxAndAddSig(t, txs) + _, err := txs.OnError(sig, 10*time.Second, Errored, 0) + require.NoError(t, err) + + // Call OnReorg + _, err = txs.OnReorg(sig, txID) + require.Error(t, err) + require.Equal(t, ErrTransactionNotFound, err) + }) + + t.Run("fail to reset non-existent transaction", func(t *testing.T) { + _, err := txs.OnReorg(randomSignature(t), "non-existent") + require.Error(t, err) + require.Equal(t, ErrTransactionNotFound, err) + }) +} + +func TestPendingTxContext_GetSignatureInfo(t *testing.T) { + t.Parallel() + // Initialize a new pendingTxContext + txs := newPendingTxContext() + t.Run("successfully retrieve existing signature info", func(t *testing.T) { + txID, sig := createTxAndAddSig(t, txs) + // Retrieve the signature info + info, err := txs.GetSignatureInfo(sig) + require.NoError(t, err) + require.Equal(t, txID, info.id) + require.Equal(t, Broadcasted, info.state) + }) + + t.Run("fail to retrieve non-existent signature info", func(t *testing.T) { + nonExistentSig := randomSignature(t) + + // Attempt to retrieve info for a signature that doesn't exist + _, err := txs.GetSignatureInfo(nonExistentSig) + require.ErrorIs(t, err, ErrSigDoesNotExist) + }) + + t.Run("concurrent access to GetSignatureInfo", func(t *testing.T) { + txID, sig := createTxAndAddSig(t, txs) + + // Perform concurrent reads + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + info, err := txs.GetSignatureInfo(sig) + require.NoError(t, err) + require.Equal(t, txID, info.id) + }() + } + wg.Wait() + }) +} + +func TestPendingTxContext_TxHasReorg(t *testing.T) { + t.Parallel() + txs := newPendingTxContext() + cancelFunc := func() {} + t.Run("no reorg: tx does not exist", func(t *testing.T) { + hasReorg := txs.TxHasReorg("non-existent") + require.False(t, hasReorg, "expected no reorg for non-existent transaction") + }) + + t.Run("no reorg: a signature >= transaction state", func(t *testing.T) { + // Create transaction and add signatures + txID, sig1 := createTxAndAddSig(t, txs) + sig2 := randomSignature(t) + require.NoError(t, txs.AddSignature(cancelFunc, txID, sig2)) + + // Transition transaction to Confirmed through sig1 + _, err := txs.OnProcessed(sig1) + require.NoError(t, err) + _, err = txs.OnConfirmed(sig1) + require.NoError(t, err) + + // sig1 is Confirmed and sig2 is Broadcasted. + // TxHasReorg should return false because sig1 >= tx state = Confirmed + hasReorg := txs.TxHasReorg(txID) + require.False(t, hasReorg, "expected no reorg when all signatures are >= transaction state") + }) + + t.Run("reorg: all signatures < transaction state", func(t *testing.T) { + // Create transaction and add signatures + txID, sig1 := createTxAndAddSig(t, txs) + sig2 := randomSignature(t) + require.NoError(t, txs.AddSignature(cancelFunc, txID, sig2)) + + // Transition transaction to Confirmed through sig1 + _, err := txs.OnProcessed(sig1) + require.NoError(t, err) + _, err = txs.OnConfirmed(sig1) + require.NoError(t, err) + + // Regress sig1 to processed state + require.NoError(t, txs.UpdateSignatureStatus(sig1, Processed)) + + // Now, sig1 is in Processed state and sig2 is in Broadcasted state. + // TxHasReorg should return true because all sigs are < transaction state = Confirmed + hasReorg := txs.TxHasReorg(txID) + require.True(t, hasReorg, "expected reorg when all signatures are < transaction state") + }) +} diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 4466b18fc..f39b75860 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -550,10 +550,15 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status // Check if the sig status has regressed to indicate a re-org. // A regression is identified when the state transitions as follows: - // - Confirmed -> Processed || Not Found - // - Processed -> Not Found + // - Confirmed -> Processed || Broadcasted || Not Found + // - Processed -> Broadcasted || Not Found currentTxState := convertStatus(status) if regressionType, isRegressed := isStatusRegression(txInfo.state, currentTxState); isRegressed { + if err := txm.txs.UpdateSignatureStatus(sig, currentTxState); err != nil { + txm.lggr.Errorw("failed to update sig status", "signature", sig, "error", err) + return + } + // Determine if the sig regression affects the transaction state. // If the tx isn't considered re-orged, skip further processing. // Multiple signatures may be in-flight for a single transaction, so a re-org @@ -564,7 +569,7 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status txm.lggr.Warnw("re-org detected for transaction", "txID", txInfo.id, "signature", sig, "previousStatus", txInfo.state, "currentStatus", currentTxState) // update the in-memory state and return the transaction associated with the signature for rebroadcasting and restarting retry/bump cycle if needed - pTx, err := txm.txs.OnReorg(sig) + pTx, err := txm.txs.OnReorg(sig, txInfo.id) if err != nil { txm.lggr.Errorw("failed to handle re-org", "signature", sig, "id", pTx.id, "error", err) return @@ -577,6 +582,7 @@ func (txm *Txm) handleReorg(ctx context.Context, sig solanaGo.Signature, status go func() { defer txm.done.Done() txm.retryTx(retryCtx, cancel, pTx, pTx.tx, sig) + txm.lggr.Debugw("re-org retry completed", "id", pTx.id) }() } // For regressions from "Processed" do not restart the cycle immediately. diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index b0ac544b7..0e96445d7 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1631,5 +1631,309 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { require.Equal(t, types.Failed, status) require.Equal(t, 1, callCount-1) // -1 because the first call is not a rebroadcast }) +} + +func TestTxm_SingleSigOnReorg(t *testing.T) { + t.Parallel() + estimator := "fixed" + id := "mocknet-" + estimator + "-" + uuid.NewString() + cfg := config.NewDefault() + cfg.Chain.FeeEstimatorMode = &estimator + cfg.Chain.TxConfirmTimeout = relayconfig.MustNewDuration(5 * time.Second) + // Enable retention to keep transactions after finality and be able to check their statuses. + cfg.Chain.TxRetentionTimeout = relayconfig.MustNewDuration(10 * time.Second) + lggr := logger.Test(t) + ctx := tests.Context(t) + + // Helper function to set up common test environment + setupTxmTest := func( + txExpirationRebroadcast bool, + latestBlockhashFunc func() (*rpc.GetLatestBlockhashResult, error), + getLatestBlockFunc func() (*rpc.GetBlockResult, error), + sendTxFunc func() (solana.Signature, error), + statuses map[solana.Signature]func() *rpc.SignatureStatusesResult, + ) (*Txm, *mocks.ReaderWriter, *keyMocks.SimpleKeystore) { + cfg.Chain.TxExpirationRebroadcast = &txExpirationRebroadcast + + mc := mocks.NewReaderWriter(t) + if latestBlockhashFunc != nil { + mc.On("LatestBlockhash", mock.Anything).Return( + func(_ context.Context) (*rpc.GetLatestBlockhashResult, error) { + return latestBlockhashFunc() + }, + ).Maybe() + } + if getLatestBlockFunc != nil { + mc.On("GetLatestBlock", mock.Anything).Return( + func(_ context.Context) (*rpc.GetBlockResult, error) { + return getLatestBlockFunc() + }, + ).Maybe() + } + if sendTxFunc != nil { + mc.On("SendTx", mock.Anything, mock.Anything).Return( + func(_ context.Context, _ *solana.Transaction) (solana.Signature, error) { + return sendTxFunc() + }, + ).Maybe() + } + mc.On("SimulateTx", mock.Anything, mock.Anything, mock.Anything).Return(&rpc.SimulateTransactionResult{}, nil).Maybe() + if statuses != nil { + mc.On("SignatureStatuses", mock.Anything, mock.AnythingOfType("[]solana.Signature")).Return( + func(_ context.Context, sigs []solana.Signature) ([]*rpc.SignatureStatusesResult, error) { + var out []*rpc.SignatureStatusesResult + for _, sig := range sigs { + getStatus, exists := statuses[sig] + if !exists { + out = append(out, nil) + } else { + out = append(out, getStatus()) + } + } + return out, nil + }, + ).Maybe() + } + + mkey := keyMocks.NewSimpleKeystore(t) + mkey.On("Sign", mock.Anything, mock.Anything, mock.Anything).Return([]byte{}, nil) + + loader := utils.NewLazyLoad(func() (client.ReaderWriter, error) { return mc, nil }) + txm := NewTxm(id, loader, nil, cfg, mkey, lggr) + require.NoError(t, txm.Start(ctx)) + t.Cleanup(func() { require.NoError(t, txm.Close()) }) + + return txm, mc, mkey + } + + // tracking prom metrics + prom := soltxmProm{id: id} + + t.Run("regressing from confirmed state restarts retry/bumping cycle", func(t *testing.T) { + statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} + + latestBlockhashFunc := func() (*rpc.GetLatestBlockhashResult, error) { + return &rpc.GetLatestBlockhashResult{ + Value: &rpc.LatestBlockhashResult{ + LastValidBlockHeight: uint64(2000), + }, + }, nil + } + + sig1 := randomSignature(t) + sendTxFunc := func() (solana.Signature, error) { + return sig1, nil + } + + var wg sync.WaitGroup + statusCallCount := 0 + wg.Add(1) + statuses[sig1] = func() *rpc.SignatureStatusesResult { + defer func() { statusCallCount++ }() + if statusCallCount < 1 { + // Initially, transaction is Processed + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } + + if statusCallCount < 3 { + // Transaction should be confirmed + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, + } + } + + if statusCallCount < 5 { + // Simulate reorg: transaction status regresses to NotFound + return nil // Status is nil (NotFound) + } + + if statusCallCount < 7 { + // Transaction should be processed again + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } + + if statusCallCount < 9 { + // Transaction should be confirmed again + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, + } + } + + // Transaction should be finalized + wg.Done() + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusFinalized, + } + } + + txm, _, mkey := setupTxmTest(false, latestBlockhashFunc, nil, sendTxFunc, statuses) + tx, _ := getTx(t, 0, mkey) + txID := "test-reorg-from-confirmed" + assert.NoError(t, txm.Enqueue(ctx, t.Name(), tx, &txID)) + wg.Wait() + waitFor(t, txm.cfg.TxConfirmTimeout(), txm, prom, empty) + + // check prom metric + prom.confirmed++ + prom.confirmed++ + prom.finalized++ + prom.assertEqual(t) + + // Check that transaction for txID has been finalized + status, err := txm.GetTransactionStatus(ctx, txID) + require.NoError(t, err) + require.Equal(t, types.Finalized, status) + }) + + t.Run("regressing from processed state does not restart retry/bumping cycle", func(t *testing.T) { + statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} + + latestBlockhashFunc := func() (*rpc.GetLatestBlockhashResult, error) { + return &rpc.GetLatestBlockhashResult{ + Value: &rpc.LatestBlockhashResult{ + LastValidBlockHeight: uint64(2000), + }, + }, nil + } + + sig1 := randomSignature(t) + sendTxFunc := func() (solana.Signature, error) { + return sig1, nil + } + + statusCallCount := 0 + var wg sync.WaitGroup + wg.Add(1) + statuses[sig1] = func() *rpc.SignatureStatusesResult { + defer func() { statusCallCount++ }() + if statusCallCount == 0 { + // Initially, transaction is Processed + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } + if statusCallCount == 1 { + wg.Done() + } + // Simulate reorg: transaction status regresses to NotFound (nil) + return nil + } + + txm, _, mkey := setupTxmTest(false, latestBlockhashFunc, nil, sendTxFunc, statuses) + tx, _ := getTx(t, 0, mkey) + txID := "test-reorg-from-processed-without-rebroadcast" + assert.NoError(t, txm.Enqueue(ctx, t.Name(), tx, &txID)) + wg.Wait() + waitFor(t, txm.cfg.TxConfirmTimeout(), txm, prom, empty) + + // check prom metric + // Transaction should be dropped after reorg and not rebroadcasted when expirationRebroadcast is off + prom.error++ + prom.drop++ + prom.assertEqual(t) + + // Check that transaction for txID has failed + status, err := txm.GetTransactionStatus(ctx, txID) + require.NoError(t, err) + require.Equal(t, types.Failed, status) + }) + + t.Run("regressing from processed state rebroadcasts tx on expiration when enabled", func(t *testing.T) { + statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} + + getLatestBlockFunc := func() (*rpc.GetBlockResult, error) { + val := uint64(1500) + return &rpc.GetBlockResult{ + BlockHeight: &val, + }, nil + } + latestBlockhashCallCount := 0 + latestBlockhashFunc := func() (*rpc.GetLatestBlockhashResult, error) { + defer func() { latestBlockhashCallCount++ }() + if latestBlockhashCallCount < 1 { + // To force rebroadcast, first call needs to be smaller than blockHeight + return &rpc.GetLatestBlockhashResult{ + Value: &rpc.LatestBlockhashResult{ + LastValidBlockHeight: uint64(1000), + }, + }, nil + } + // following rebroadcast call will go through because lastValidBlockHeight is bigger than blockHeight + return &rpc.GetLatestBlockhashResult{ + Value: &rpc.LatestBlockhashResult{ + LastValidBlockHeight: uint64(2000), + }, + }, nil + } + + sig1 := randomSignature(t) + sendTxFunc := func() (solana.Signature, error) { + return sig1, nil + } + + statusCallCount, statusCallRebroadcastCount := 0, 0 + nowTs := time.Now() + var wg sync.WaitGroup + wg.Add(1) + statuses[sig1] = func() *rpc.SignatureStatusesResult { + defer func() { statusCallCount++ }() + + // Initially, transaction is Processed + if statusCallCount == 0 { + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } + + // we get regression after first call + if time.Since(nowTs) < cfg.TxConfirmTimeout()-2*time.Second { + return nil + } + + // Transaction should be rebroadcasted and go through each state after expiration rebroadcast + if statusCallRebroadcastCount == 0 { + statusCallRebroadcastCount++ + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusProcessed, + } + } + + if statusCallRebroadcastCount == 1 { + statusCallRebroadcastCount++ + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusConfirmed, + } + } + + wg.Done() + return &rpc.SignatureStatusesResult{ + ConfirmationStatus: rpc.ConfirmationStatusFinalized, + } + } + + txm, _, mkey := setupTxmTest(true, latestBlockhashFunc, getLatestBlockFunc, sendTxFunc, statuses) + tx, _ := getTx(t, 0, mkey) + txID := "test-reorg-from-processed-with-rebroadcast" + assert.NoError(t, txm.Enqueue(ctx, t.Name(), tx, &txID)) + wg.Wait() + waitFor(t, txm.cfg.TxConfirmTimeout(), txm, prom, empty) + + // check prom metric + // Transaction should be rebroadcasted and finalized + prom.confirmed++ + prom.finalized++ + prom.assertEqual(t) + prom.assertEqual(t) + + // Check that transaction for txID has been finalized and rebroadcasted 1 time. + status, err := txm.GetTransactionStatus(ctx, txID) + require.NoError(t, err) + require.Equal(t, types.Finalized, status) + require.Equal(t, 1, latestBlockhashCallCount-1) // -1 because the first call is not a rebroadcast + }) } diff --git a/pkg/solana/txm/utils.go b/pkg/solana/txm/utils.go index 3373bd60c..7e83a4e74 100644 --- a/pkg/solana/txm/utils.go +++ b/pkg/solana/txm/utils.go @@ -59,19 +59,17 @@ const ( ) // isStatusRegression checks if the current status is a regression compared to the previous status: -// - Confirmed -> Processed, Broadcasted: should not regress -// - Processed -> Broadcasted: should not regress +// - Confirmed -> Processed, Broadcasted, Not Found: should not regress +// - Processed -> Broadcasted, Not Found: should not regress // Returns true if a regression is detected, indicating a possible re-org. func isStatusRegression(previous, current TxState) (regressionType, bool) { switch previous { case Confirmed: - // Confirmed transactions should not regress to Processed or Broadcasted. - if current == Processed || current == Broadcasted { + if current == Processed || current == Broadcasted || current == NotFound { return FromConfirmed, true } case Processed: - // Processed transactions should not regress to Broadcasted. - if current == Broadcasted { + if current == Broadcasted || current == NotFound { return FromProcessed, true } default: From 10e5d9dec9e6b789df7a8f92c6b1a296b5ed5715 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Tue, 10 Dec 2024 12:54:35 -0300 Subject: [PATCH 28/31] address feedback --- pkg/solana/txm/pendingtx.go | 8 ++++---- pkg/solana/txm/txm.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/solana/txm/pendingtx.go b/pkg/solana/txm/pendingtx.go index 70a1aee3a..8e2507d9a 100644 --- a/pkg/solana/txm/pendingtx.go +++ b/pkg/solana/txm/pendingtx.go @@ -606,10 +606,10 @@ func (c *pendingTxContext) OnReorg(sig solana.Signature, id string) (pendingTx, } // Reset the transaction state to 'Broadcasted' upon detecting a reorg. - // Even if the transaction might have already progressed to 'Processed' before the reorg, - // we reset it to 'Broadcasted' for simplicity. - // Any state advancements (e.g., moving to 'Processed' or 'Confirmed') will be picked up - // on the next status polling cycle. + // Even if the transaction might have already progressed to 'Processed' after the reorg, + // we reset it to 'Broadcasted' for simplicity here. + // Any state advancements (e.g., moving to 'Processed') will be picked up + // by the current status polling cycle after handling the reorg. // This approach does not introduce any risk with the expiration logic since // we check for status changes before considering a transaction for expiration. pTx.state = Broadcasted diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 89b08faf9..36ab12d97 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -459,11 +459,11 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr for j := 0; j < len(sortedRes); j++ { sig, status := sortedSigs[j], sortedRes[j] - // sig not found could mean invalid tx or not picked up yet, keep polling if status == nil { - txm.handleNotFoundSignatureStatus(sig) - // check if a potential re-org has occurred for this sig and handle it + // sig not found could mean invalid tx or not picked up yet, keep polling + // we also need to check if a potential re-org has occurred for this sig and handle it txm.handleReorg(ctx, sig, status) + txm.handleNotFoundSignatureStatus(sig) continue } @@ -477,8 +477,8 @@ func (txm *Txm) processConfirmations(ctx context.Context, client client.ReaderWr case rpc.ConfirmationStatusProcessed: // if signature is processed, keep polling for confirmed or finalized status // we also need to check if a potential re-org has occurred for this sig and handle it - txm.handleProcessedSignatureStatus(sig) txm.handleReorg(ctx, sig, status) + txm.handleProcessedSignatureStatus(sig) case rpc.ConfirmationStatusConfirmed: // if signature is confirmed, keep polling for finalized status txm.handleConfirmedSignatureStatus(sig) From 17769f735fb5d9b96bf32e8edb565fd8930e9573 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 12 Dec 2024 18:19:38 -0300 Subject: [PATCH 29/31] get height instead of whole block optimization --- pkg/solana/client/client.go | 15 +++++++ pkg/solana/client/client_test.go | 32 ++++++++++++++ pkg/solana/client/mocks/reader_writer.go | 56 ++++++++++++++++++++++++ pkg/solana/txm/txm.go | 8 ++-- 4 files changed, 107 insertions(+), 4 deletions(-) diff --git a/pkg/solana/client/client.go b/pkg/solana/client/client.go index a015fdc1f..5eaa37b89 100644 --- a/pkg/solana/client/client.go +++ b/pkg/solana/client/client.go @@ -36,6 +36,8 @@ type Reader interface { ChainID(ctx context.Context) (mn.StringID, error) GetFeeForMessage(ctx context.Context, msg string) (uint64, error) GetLatestBlock(ctx context.Context) (*rpc.GetBlockResult, error) + // GetLatestBlockHeight returns the latest block height of the node based on the configured commitment type + GetLatestBlockHeight(ctx context.Context) (uint64, error) GetTransaction(ctx context.Context, txHash solana.Signature, opts *rpc.GetTransactionOpts) (*rpc.GetTransactionResult, error) GetBlocks(ctx context.Context, startSlot uint64, endSlot *uint64) (rpc.BlocksResult, error) GetBlocksWithLimit(ctx context.Context, startSlot uint64, limit uint64) (*rpc.BlocksResult, error) @@ -331,6 +333,19 @@ func (c *Client) GetLatestBlock(ctx context.Context) (*rpc.GetBlockResult, error return v.(*rpc.GetBlockResult), err } +// GetLatestBlockHeight returns the latest block height of the node based on the configured commitment type +func (c *Client) GetLatestBlockHeight(ctx context.Context) (uint64, error) { + done := c.latency("latest_block_height") + defer done() + ctx, cancel := context.WithTimeout(ctx, c.txTimeout) + defer cancel() + + v, err, _ := c.requestGroup.Do("GetBlockHeight", func() (interface{}, error) { + return c.rpc.GetBlockHeight(ctx, c.commitment) + }) + return v.(uint64), err +} + func (c *Client) GetBlock(ctx context.Context, slot uint64) (*rpc.GetBlockResult, error) { // get block based on slot done := c.latency("get_block") diff --git a/pkg/solana/client/client_test.go b/pkg/solana/client/client_test.go index 8149b0839..6ca3c1727 100644 --- a/pkg/solana/client/client_test.go +++ b/pkg/solana/client/client_test.go @@ -125,6 +125,12 @@ func TestClient_Reader_Integration(t *testing.T) { assert.GreaterOrEqual(t, slot, startSlot) assert.LessOrEqual(t, slot, slot0) } + + // GetLatestBlockHeight + // Test fetching the latest block height + blockHeight, err := c.GetLatestBlockHeight(ctx) + require.NoError(t, err) + require.Greater(t, blockHeight, uint64(0), "Block height should be greater than 0") } func TestClient_Reader_ChainID(t *testing.T) { @@ -288,6 +294,32 @@ func TestClient_GetBlocks(t *testing.T) { requestTimeout, 500*time.Millisecond) } +func TestClient_GetLatestBlockHeight(t *testing.T) { + t.Parallel() + + ctx := tests.Context(t) + url := SetupLocalSolNode(t) + requestTimeout := 5 * time.Second + lggr := logger.Test(t) + cfg := config.NewDefault() + + // Initialize the client + c, err := NewClient(url, cfg, requestTimeout, lggr) + require.NoError(t, err) + + // Get the latest block height + blockHeight, err := c.GetLatestBlockHeight(ctx) + require.NoError(t, err) + require.Greater(t, blockHeight, uint64(0), "Block height should be greater than 0") + + // Wait until the block height increases + require.Eventually(t, func() bool { + newBlockHeight, err := c.GetLatestBlockHeight(ctx) + require.NoError(t, err) + return newBlockHeight > blockHeight + }, 10*time.Second, 1*time.Second, "Block height should eventually increase") +} + func TestClient_SendTxDuplicates_Integration(t *testing.T) { ctx := tests.Context(t) // set up environment diff --git a/pkg/solana/client/mocks/reader_writer.go b/pkg/solana/client/mocks/reader_writer.go index c64a4a9ad..7c72ca183 100644 --- a/pkg/solana/client/mocks/reader_writer.go +++ b/pkg/solana/client/mocks/reader_writer.go @@ -492,6 +492,62 @@ func (_c *ReaderWriter_GetLatestBlock_Call) RunAndReturn(run func(context.Contex return _c } +// GetLatestBlockHeight provides a mock function with given fields: ctx +func (_m *ReaderWriter) GetLatestBlockHeight(ctx context.Context) (uint64, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetLatestBlockHeight") + } + + var r0 uint64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (uint64, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) uint64); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(uint64) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ReaderWriter_GetLatestBlockHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLatestBlockHeight' +type ReaderWriter_GetLatestBlockHeight_Call struct { + *mock.Call +} + +// GetLatestBlockHeight is a helper method to define mock.On call +// - ctx context.Context +func (_e *ReaderWriter_Expecter) GetLatestBlockHeight(ctx interface{}) *ReaderWriter_GetLatestBlockHeight_Call { + return &ReaderWriter_GetLatestBlockHeight_Call{Call: _e.mock.On("GetLatestBlockHeight", ctx)} +} + +func (_c *ReaderWriter_GetLatestBlockHeight_Call) Run(run func(ctx context.Context)) *ReaderWriter_GetLatestBlockHeight_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *ReaderWriter_GetLatestBlockHeight_Call) Return(_a0 uint64, _a1 error) *ReaderWriter_GetLatestBlockHeight_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ReaderWriter_GetLatestBlockHeight_Call) RunAndReturn(run func(context.Context) (uint64, error)) *ReaderWriter_GetLatestBlockHeight_Call { + _c.Call.Return(run) + return _c +} + // GetSignaturesForAddressWithOpts provides a mock function with given fields: ctx, addr, opts func (_m *ReaderWriter) GetSignaturesForAddressWithOpts(ctx context.Context, addr solana.PublicKey, opts *rpc.GetSignaturesForAddressOpts) ([]*rpc.TransactionSignature, error) { ret := _m.Called(ctx, addr, opts) diff --git a/pkg/solana/txm/txm.go b/pkg/solana/txm/txm.go index 974b8140d..59fc1a9e7 100644 --- a/pkg/solana/txm/txm.go +++ b/pkg/solana/txm/txm.go @@ -640,14 +640,14 @@ func (txm *Txm) handleFinalizedSignatureStatus(sig solanaGo.Signature) { // An expired tx is one where it's blockhash lastValidBlockHeight (last valid block number) is smaller than the current block height (block number). // If any error occurs during rebroadcast attempt, they are discarded, and the function continues with the next transaction. func (txm *Txm) rebroadcastExpiredTxs(ctx context.Context, client client.ReaderWriter) { - currBlock, err := client.GetLatestBlock(ctx) - if err != nil || currBlock == nil || currBlock.BlockHeight == nil { + blockHeight, err := client.GetLatestBlockHeight(ctx) + if err != nil || blockHeight == 0 { txm.lggr.Errorw("failed to get current block height", "error", err) return } // Rebroadcast all expired txes using currBlockHeight (current block number) - for _, tx := range txm.txs.ListAllExpiredBroadcastedTxs(*currBlock.BlockHeight) { - txm.lggr.Debugw("transaction expired, rebroadcasting", "id", tx.id, "signature", tx.signatures, "lastValidBlockHeight", tx.lastValidBlockHeight, "currentBlockHeight", *currBlock.BlockHeight) + for _, tx := range txm.txs.ListAllExpiredBroadcastedTxs(blockHeight) { + txm.lggr.Debugw("transaction expired, rebroadcasting", "id", tx.id, "signature", tx.signatures, "lastValidBlockHeight", tx.lastValidBlockHeight, "currentBlockHeight", blockHeight) // Removes all signatures associated to tx and cancels context. _, err := txm.txs.Remove(tx.id) if err != nil { From d6fb89194aae57aa701401603fa00a8f7f86d5ef Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 12 Dec 2024 18:53:06 -0300 Subject: [PATCH 30/31] fix mocks on expiration --- pkg/solana/txm/txm_internal_test.go | 46 +++++++++++------------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index 4d37881d0..fccf1eab4 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1225,7 +1225,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { setupTxmTest := func( txExpirationRebroadcast bool, latestBlockhashFunc func() (*rpc.GetLatestBlockhashResult, error), - getLatestBlockFunc func() (*rpc.GetBlockResult, error), + getLatestBlockHeightFunc func() (uint64, error), sendTxFunc func() (solana.Signature, error), statuses map[solana.Signature]func() *rpc.SignatureStatusesResult, ) (*Txm, *mocks.ReaderWriter, *keyMocks.SimpleKeystore) { @@ -1239,10 +1239,10 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { }, ).Maybe() } - if getLatestBlockFunc != nil { - mc.On("GetLatestBlock", mock.Anything).Return( - func(_ context.Context) (*rpc.GetBlockResult, error) { - return getLatestBlockFunc() + if getLatestBlockHeightFunc != nil { + mc.On("GetLatestBlockHeight", mock.Anything).Return( + func(_ context.Context) (uint64, error) { + return getLatestBlockHeightFunc() }, ).Maybe() } @@ -1291,11 +1291,8 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} // Mock getLatestBlock to return a value greater than 0 for blockHeight - getLatestBlockFunc := func() (*rpc.GetBlockResult, error) { - val := uint64(1500) - return &rpc.GetBlockResult{ - BlockHeight: &val, - }, nil + getLatestBlockHeightFunc := func() (uint64, error) { + return 1500, nil } callCount := 0 @@ -1348,7 +1345,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { } } - txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockFunc, sendTxFunc, statuses) + txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockHeightFunc, sendTxFunc, statuses) tx, _ := getTx(t, 0, mkey) txID := "test-rebroadcast" @@ -1425,11 +1422,8 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} // Mock getLatestBlock to return a value greater than 0 - getLatestBlockFunc := func() (*rpc.GetBlockResult, error) { - val := uint64(1500) - return &rpc.GetBlockResult{ - BlockHeight: &val, - }, nil + getLatestBlockHeightFunc := func() (uint64, error) { + return 1500, nil } // Mock LatestBlockhash to return an invalid blockhash in the first 3 attempts (initial + 2 rebroadcasts) @@ -1482,7 +1476,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { } } - txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockFunc, sendTxFunc, statuses) + txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockHeightFunc, sendTxFunc, statuses) tx, _ := getTx(t, 0, mkey) txID := "test-rebroadcast" assert.NoError(t, txm.Enqueue(ctx, t.Name(), tx, &txID)) @@ -1510,11 +1504,8 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { } // Mock getLatestBlock to return a value greater than 0 - getLatestBlockFunc := func() (*rpc.GetBlockResult, error) { - val := uint64(1500) - return &rpc.GetBlockResult{ - BlockHeight: &val, - }, nil + getLatestBlockHeightFunc := func() (uint64, error) { + return 1500, nil } callCount := 0 @@ -1547,7 +1538,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { return out } - txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockFunc, sendTxFunc, statuses) + txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockHeightFunc, sendTxFunc, statuses) tx, _ := getTx(t, 0, mkey) txID := "test-confirmed-before-rebroadcast" assert.NoError(t, txm.Enqueue(ctx, t.Name(), tx, &txID)) @@ -1572,11 +1563,8 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { // To force rebroadcast, first call needs to be smaller than blockHeight // following rebroadcast call will go through because lastValidBlockHeight will be bigger than blockHeight - getLatestBlockFunc := func() (*rpc.GetBlockResult, error) { - val := uint64(1500) - return &rpc.GetBlockResult{ - BlockHeight: &val, - }, nil + getLatestBlockHeightFunc := func() (uint64, error) { + return 1500, nil } callCount := 0 @@ -1613,7 +1601,7 @@ func TestTxm_ExpirationRebroadcast(t *testing.T) { return nil } - txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockFunc, sendTxFunc, statuses) + txm, _, mkey := setupTxmTest(txExpirationRebroadcast, latestBlockhashFunc, getLatestBlockHeightFunc, sendTxFunc, statuses) tx, _ := getTx(t, 0, mkey) txID := "test-rebroadcast-error" assert.NoError(t, txm.Enqueue(ctx, t.Name(), tx, &txID)) From b6f47297f4c6c5bc8431c5e6aa9ade6fb6009b70 Mon Sep 17 00:00:00 2001 From: Farber98 Date: Thu, 12 Dec 2024 20:20:34 -0300 Subject: [PATCH 31/31] fix test --- pkg/solana/txm/txm_internal_test.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pkg/solana/txm/txm_internal_test.go b/pkg/solana/txm/txm_internal_test.go index fccf1eab4..2cbc4c53c 100644 --- a/pkg/solana/txm/txm_internal_test.go +++ b/pkg/solana/txm/txm_internal_test.go @@ -1637,7 +1637,7 @@ func TestTxm_SingleSigOnReorg(t *testing.T) { setupTxmTest := func( txExpirationRebroadcast bool, latestBlockhashFunc func() (*rpc.GetLatestBlockhashResult, error), - getLatestBlockFunc func() (*rpc.GetBlockResult, error), + getLatestBlockHeightFunc func() (uint64, error), sendTxFunc func() (solana.Signature, error), statuses map[solana.Signature]func() *rpc.SignatureStatusesResult, ) (*Txm, *mocks.ReaderWriter, *keyMocks.SimpleKeystore) { @@ -1651,10 +1651,10 @@ func TestTxm_SingleSigOnReorg(t *testing.T) { }, ).Maybe() } - if getLatestBlockFunc != nil { - mc.On("GetLatestBlock", mock.Anything).Return( - func(_ context.Context) (*rpc.GetBlockResult, error) { - return getLatestBlockFunc() + if getLatestBlockHeightFunc != nil { + mc.On("GetLatestBlockHeight", mock.Anything).Return( + func(_ context.Context) (uint64, error) { + return getLatestBlockHeightFunc() }, ).Maybe() } @@ -1833,11 +1833,8 @@ func TestTxm_SingleSigOnReorg(t *testing.T) { t.Run("regressing from processed state rebroadcasts tx on expiration when enabled", func(t *testing.T) { statuses := map[solana.Signature]func() *rpc.SignatureStatusesResult{} - getLatestBlockFunc := func() (*rpc.GetBlockResult, error) { - val := uint64(1500) - return &rpc.GetBlockResult{ - BlockHeight: &val, - }, nil + getLatestBlockHeightFunc := func() (uint64, error) { + return 1500, nil } latestBlockhashCallCount := 0 @@ -1904,7 +1901,7 @@ func TestTxm_SingleSigOnReorg(t *testing.T) { } } - txm, _, mkey := setupTxmTest(true, latestBlockhashFunc, getLatestBlockFunc, sendTxFunc, statuses) + txm, _, mkey := setupTxmTest(true, latestBlockhashFunc, getLatestBlockHeightFunc, sendTxFunc, statuses) tx, _ := getTx(t, 0, mkey) txID := "test-reorg-from-processed-with-rebroadcast" assert.NoError(t, txm.Enqueue(ctx, t.Name(), tx, &txID))