fix(thorchain/utxo): fix race conditions (#1281)

chain/thorchain/sidecar.go

@@ -71,6 +71,10 @@ func NewSidecar(
 		homeDir = "/home/sidecar"
 	}
 
+	// Give each sidecard their own env copy for runtime changes
+	envCopy := make([]string, len(env))
+	copy(envCopy, env)
+
 	s := &SidecarProcess{
 		log:              log,
 		Index:            index,
@@ -85,7 +89,7 @@ func NewSidecar(
 		homeDir:          homeDir,
 		ports:            processPorts,
 		startCmd:         startCmd,
-		env:              env,
+		env:              envCopy,
 	}
 	s.containerLifecycle = dockerutil.NewContainerLifecycle(log, dockerClient, s.Name())
 

chain/utxo/cli.go

@@ -109,10 +109,12 @@ func (c *UtxoChain) CreateWallet(ctx context.Context, keyName string) error {
 			return err
 		}
 
+		c.MapAccess.Lock()
 		c.KeyNameToWalletMap[keyName] = &NodeWallet{
 			keyName:   keyName,
 			loadCount: 1,
 		}
+		c.MapAccess.Unlock()
 	}
 
 	return c.UnloadWallet(ctx, keyName)
@@ -151,12 +153,14 @@ func (c *UtxoChain) GetNewAddress(ctx context.Context, keyName string, mweb bool
 		addr = splitAddr[1]
 	}
 
+	c.MapAccess.Lock()
 	wallet.address = addr
 	c.AddrToKeyNameMap[addr] = keyName
 
 	if c.WalletVersion >= noDefaultKeyWalletVersion {
 		wallet.ready = true
 	}
+	c.MapAccess.Unlock()
 
 	if err := c.UnloadWallet(ctx, keyName); err != nil {
 		return "", err

chain/utxo/utxo_chain.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	dockertypes "github.com/docker/docker/api/types"
@@ -40,6 +41,7 @@ var natPorts = nat.PortMap{
 type UtxoChain struct {
 	testName string
 	cfg      ibc.ChainConfig
+	cancel   context.CancelFunc
 
 	log *zap.Logger
 
@@ -60,6 +62,9 @@ type UtxoChain struct {
 	AddrToKeyNameMap   map[string]string
 	KeyNameToWalletMap map[string]*NodeWallet
 
+	// Mutex for reading/writing AddrToKeyNameMap and KeyNameToWalletMap
+	MapAccess sync.Mutex
+
 	WalletVersion        int
 	unloadWalletAfterUse bool
 }
@@ -259,52 +264,6 @@ func (c *UtxoChain) Start(testName string, ctx context.Context, additionalGenesi
 	// Wait for rpc to come up
 	time.Sleep(time.Second * 5)
 
-	go func() {
-		ctx := context.Background()
-		amount := "100"
-		nextBlockHeight := 100
-		if c.cfg.CoinType == "3" {
-			amount = "1000" // Dogecoin needs more blocks for more coins
-			nextBlockHeight = 1000
-		}
-		for {
-			faucetWallet, found := c.KeyNameToWalletMap[faucetKeyName]
-			if !found || !faucetWallet.ready {
-				time.Sleep(time.Second)
-				continue
-			}
-
-			// If faucet exists, chain is up and running. Any future error should return from this go routine.
-			// If the chain stops, we will then error and return from this go routine
-			// Don't use ctx from Start(), it gets cancelled soon after returning.
-			cmd = append(c.BaseCli, "generatetoaddress", amount, faucetWallet.address)
-			_, _, err = c.Exec(ctx, cmd, nil)
-			if err != nil {
-				c.logger().Error("generatetoaddress error", zap.Error(err))
-				return
-			}
-			amount = "1"
-			if nextBlockHeight == 431 && c.cfg.CoinType == "2" {
-				keyName := "mweb"
-				if err := c.CreateWallet(ctx, keyName); err != nil {
-					c.logger().Error("error creating mweb wallet at block 431", zap.String("chain", c.cfg.ChainID), zap.Error(err))
-					return
-				}
-				addr, err := c.GetNewAddress(ctx, keyName, true)
-				if err != nil {
-					c.logger().Error("error creating mweb wallet at block 431", zap.String("chain", c.cfg.ChainID), zap.Error(err))
-					return
-				}
-				if err := c.sendToMwebAddress(ctx, faucetKeyName, addr, 1); err != nil {
-					c.logger().Error("error sending to mweb wallet at block 431", zap.String("chain", c.cfg.ChainID), zap.Error(err))
-					return
-				}
-			}
-			nextBlockHeight++
-			time.Sleep(time.Second * 2)
-		}
-	}()
-
 	c.WalletVersion, _ = c.GetWalletVersion(ctx, "")
 
 	if err := c.CreateWallet(ctx, faucetKeyName); err != nil {
@@ -327,6 +286,63 @@ func (c *UtxoChain) Start(testName string, ctx context.Context, additionalGenesi
 		return err
 	}
 
+	go func() {
+		// Don't use ctx from Start(), it gets cancelled soon after returning.
+		goRoutineCtx := context.Background()
+		goRoutineCtx, c.cancel = context.WithCancel(goRoutineCtx)
+		amount := "100"
+		nextBlockHeight := 100
+		if c.cfg.CoinType == "3" {
+			amount = "1000" // Dogecoin needs more blocks for more coins
+			nextBlockHeight = 1000
+		}
+
+		c.MapAccess.Lock()
+		faucetWallet, found := c.KeyNameToWalletMap[faucetKeyName]
+		if !found || !faucetWallet.ready {
+			c.logger().Error("faucet wallet not found or not ready")
+			c.MapAccess.Unlock()
+			return
+		}
+		c.MapAccess.Unlock()
+
+		utxoBlockTime := time.Second * 2
+		timer := time.NewTimer(utxoBlockTime)
+		defer timer.Stop()
+		for {
+			select {
+			case <-goRoutineCtx.Done():
+				return
+			case <-timer.C:
+				cmd = append(c.BaseCli, "generatetoaddress", amount, faucetWallet.address)
+				_, _, err := c.Exec(goRoutineCtx, cmd, nil)
+				if err != nil {
+					c.logger().Error("generatetoaddress error", zap.Error(err))
+					return
+				}
+				amount = "1"
+				if nextBlockHeight == 431 && c.cfg.CoinType == "2" {
+					keyName := "mweb"
+					if err := c.CreateWallet(goRoutineCtx, keyName); err != nil {
+						c.logger().Error("error creating mweb wallet at block 431", zap.String("chain", c.cfg.ChainID), zap.Error(err))
+						return
+					}
+					addr, err := c.GetNewAddress(goRoutineCtx, keyName, true)
+					if err != nil {
+						c.logger().Error("error creating mweb wallet at block 431", zap.String("chain", c.cfg.ChainID), zap.Error(err))
+						return
+					}
+					if err := c.sendToMwebAddress(goRoutineCtx, faucetKeyName, addr, 1); err != nil {
+						c.logger().Error("error sending to mweb wallet at block 431", zap.String("chain", c.cfg.ChainID), zap.Error(err))
+						return
+					}
+				}
+				nextBlockHeight++
+				timer.Reset(utxoBlockTime)
+			}
+		}
+	}()
+
 	// Wait for 100 blocks to be created, coins mature after 100 blocks and the faucet starts getting 50 spendable coins/block onwards
 	// Then wait the standard 2 blocks which also gives the faucet a starting balance of 100 coins
 	for height, err := c.Height(ctx); err == nil && height < int64(102); {
@@ -397,6 +413,8 @@ func (c *UtxoChain) CreateKey(ctx context.Context, keyName string) error {
 
 // Get address of account, cast to a string to use.
 func (c *UtxoChain) GetAddress(ctx context.Context, keyName string) ([]byte, error) {
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	wallet, ok := c.KeyNameToWalletMap[keyName]
 	if ok {
 		return []byte(wallet.address), nil
@@ -474,10 +492,12 @@ func (c *UtxoChain) Height(ctx context.Context) (int64, error) {
 }
 
 func (c *UtxoChain) GetBalance(ctx context.Context, address string, denom string) (sdkmath.Int, error) {
+	c.MapAccess.Lock()
 	keyName, ok := c.AddrToKeyNameMap[address]
 	if !ok {
 		return sdkmath.Int{}, fmt.Errorf("wallet not found for address: %s", address)
 	}
+	c.MapAccess.Unlock()
 
 	var coinsWithDecimal float64
 	if c.WalletVersion >= noDefaultKeyWalletVersion {
@@ -534,3 +554,7 @@ func (c *UtxoChain) BuildWallet(ctx context.Context, keyName string, mnemonic st
 	}
 	return NewWallet(keyName, string(address)), nil
 }
+
+func (c *UtxoChain) Stop() {
+	c.cancel()
+}

chain/utxo/wallet.go

@@ -50,6 +50,8 @@ type NodeWallet struct {
 }
 
 func (c *UtxoChain) getWalletForNewAddress(keyName string) (*NodeWallet, error) {
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	wallet, found := c.KeyNameToWalletMap[keyName]
 	if c.WalletVersion >= noDefaultKeyWalletVersion {
 		if !found {
@@ -75,6 +77,8 @@ func (c *UtxoChain) getWalletForNewAddress(keyName string) (*NodeWallet, error)
 }
 
 func (c *UtxoChain) getWalletForSetAccount(keyName string, addr string) (*NodeWallet, error) {
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	wallet, found := c.KeyNameToWalletMap[keyName]
 	if !found {
 		return nil, fmt.Errorf("wallet keyname (%s) not found, get new address not called", keyName)
@@ -100,6 +104,8 @@ func (c *UtxoChain) getWalletForUse(keyName string) (*NodeWallet, error) {
 }
 
 func (c *UtxoChain) getWallet(keyName string) (*NodeWallet, error) {
+	c.MapAccess.Lock()
+	defer c.MapAccess.Unlock()
 	wallet, found := c.KeyNameToWalletMap[keyName]
 	if !found {
 		return nil, fmt.Errorf("wallet keyname (%s) not found", keyName)

examples/thorchain/setup_test.go

@@ -94,6 +94,12 @@ func StartExoChains(t *testing.T, ctx context.Context, client *client.Client, ne
 	}))
 	t.Cleanup(func() {
 		_ = ic.Close()
+		for _, chain := range chains {
+			utxoChain, ok := chain.(*utxo.UtxoChain)
+			if ok {
+				utxoChain.Stop()
+			}
+		}
 	})
 
 	return exoChains

examples/utxo/start_test.go

@@ -56,6 +56,12 @@ func TestUtxo(t *testing.T) {
 	}))
 	t.Cleanup(func() {
 		_ = ic.Close()
+		for _, chain := range chains {
+			utxoChain, ok := chain.(*utxo.UtxoChain)
+			if ok {
+				utxoChain.Stop()
+			}
+		}
 	})
 
 	// Create and fund a user using GetAndFundTestUsers

local-interchain/chains/state/README.md

@@ -1,3 +1,3 @@
 
 ## avs-and-eigenlayer-deployed-anvil-state.json
-- <https://github.com/mangata-finance/eigen-layer-monorepo/blob/main/tests/integration/avs-and-eigenlayer-deployed-anvil-state.json>
+- <https://github.com/gasp-xyz/gasp-monorepo/blob/v0.3.0/tests/integration/avs-and-eigenlayer-deployed-anvil-state.json>