Merge branch 'master' into handleRogueUpdates

build/version.go

@@ -47,7 +47,7 @@ const (
 
 	// AppPreRelease MUST only contain characters from semanticAlphabet
 	// per the semantic versioning spec.
-	AppPreRelease = "beta.rc2"
+	AppPreRelease = "beta.rc3"
 )
 
 func init() {

config.go

@@ -337,6 +337,9 @@ type Config struct {
 
 	Profile string `long:"profile" description:"Enable HTTP profiling on either a port or host:port"`
 
+	BlockingProfile int `long:"blockingprofile" description:"Used to enable a blocking profile to be served on the profiling port. This takes a value from 0 to 1, with 1 including every blocking event, and 0 including no events."`
+	MutexProfile    int `long:"mutexprofile" description:"Used to Enable a mutex profile to be served on the profiling port. This takes a value from 0 to 1, with 1 including every mutex event, and 0 including no events."`
+
 	UnsafeDisconnect   bool   `long:"unsafe-disconnect" description:"DEPRECATED: Allows the rpcserver to intentionally disconnect from peers with open channels. THIS FLAG WILL BE REMOVED IN 0.10.0"`
 	UnsafeReplay       bool   `long:"unsafe-replay" description:"Causes a link to replay the adds on its commitment txn after starting up, this enables testing of the sphinx replay logic."`
 	MaxPendingChannels int    `long:"maxpendingchannels" description:"The maximum number of incoming pending channels permitted per peer."`

contractcourt/channel_arbitrator.go

@@ -1050,6 +1050,20 @@ func (c *ChannelArbitrator) stateStep(
 		if err != nil {
 			log.Errorf("ChannelArbitrator(%v): unable to "+
 				"force close: %v", c.cfg.ChanPoint, err)
+
+			// We tried to force close (HTLC may be expiring from
+			// our PoV, etc), but we think we've lost data. In this
+			// case, we'll not force close, but terminate the state
+			// machine here to wait to see what confirms on chain.
+			if errors.Is(err, lnwallet.ErrForceCloseLocalDataLoss) {
+				log.Error("ChannelArbitrator(%v): broadcast "+
+					"failed due to local data loss, "+
+					"waiting for on chain confimation...",
+					c.cfg.ChanPoint)
+
+				return StateBroadcastCommit, nil, nil
+			}
+
 			return StateError, closeTx, err
 		}
 		closeTx = closeSummary.CloseTx

contractcourt/channel_arbitrator_test.go

@@ -286,17 +286,32 @@ func (c *chanArbTestCtx) Restart(restartClosure func(*chanArbTestCtx)) (*chanArb
 	return newCtx, nil
 }
 
+// testChanArbOpts is a struct that contains options that can be used to
+// initialize the channel arbitrator test context.
+type testChanArbOpts struct {
+	forceCloseErr error
+	arbCfg        *ChannelArbitratorConfig
+}
+
 // testChanArbOption applies custom settings to a channel arbitrator config for
 // testing purposes.
-type testChanArbOption func(cfg *ChannelArbitratorConfig)
+type testChanArbOption func(cfg *testChanArbOpts)
 
-// remoteInitiatorOption sets the MarkChannelClosed function in the
-// Channel Arbitrator's config.
+// remoteInitiatorOption sets the MarkChannelClosed function in the Channel
+// Arbitrator's config.
 func withMarkClosed(markClosed func(*channeldb.ChannelCloseSummary,
 	...channeldb.ChannelStatus) error) testChanArbOption {
 
-	return func(cfg *ChannelArbitratorConfig) {
-		cfg.MarkChannelClosed = markClosed
+	return func(cfg *testChanArbOpts) {
+		cfg.arbCfg.MarkChannelClosed = markClosed
+	}
+}
+
+// withForceCloseErr is used to specify an error that should be returned when
+// the channel arb tries to force close a channel.
+func withForceCloseErr(err error) testChanArbOption {
+	return func(opts *testChanArbOpts) {
+		opts.forceCloseErr = err
 	}
 }
 
@@ -386,7 +401,6 @@ func createTestChannelArbitrator(t *testing.T, log ArbitratorLog,
 			resolvedChan <- struct{}{}
 			return nil
 		},
-		Channel: &mockChannel{},
 		MarkCommitmentBroadcasted: func(_ *wire.MsgTx, _ bool) error {
 			return nil
 		},
@@ -408,9 +422,17 @@ func createTestChannelArbitrator(t *testing.T, log ArbitratorLog,
 		},
 	}
 
+	testOpts := &testChanArbOpts{
+		arbCfg: arbCfg,
+	}
+
 	// Apply all custom options to the config struct.
 	for _, option := range opts {
-		option(arbCfg)
+		option(testOpts)
+	}
+
+	arbCfg.Channel = &mockChannel{
+		forceCloseErr: testOpts.forceCloseErr,
 	}
 
 	var cleanUp func()
@@ -2686,10 +2708,11 @@ func TestChannelArbitratorAnchors(t *testing.T) {
 	)
 }
 
-// TestChannelArbitratorStartAfterCommitmentRejected tests that when we run into
-// the case where our commitment tx is rejected by our bitcoin backend we still
-// continue to startup the arbitrator for a specific set of errors.
-func TestChannelArbitratorStartAfterCommitmentRejected(t *testing.T) {
+// TestChannelArbitratorStartForceCloseFail tests that when we run into the
+// case where our commitment tx is rejected by our bitcoin backend, or we fail
+// to force close, we still continue to startup the arbitrator for a
+// specific set of errors.
+func TestChannelArbitratorStartForceCloseFail(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
@@ -2698,6 +2721,10 @@ func TestChannelArbitratorStartAfterCommitmentRejected(t *testing.T) {
 		// The specific error during broadcasting the transaction.
 		broadcastErr error
 
+		// forceCloseErr is the error returned when we try to force the
+		// channel.
+		forceCloseErr error
+
 		// expected state when the startup of the arbitrator succeeds.
 		expectedState ArbitratorState
 
@@ -2726,6 +2753,16 @@ func TestChannelArbitratorStartAfterCommitmentRejected(t *testing.T) {
 			expectedState:   StateBroadcastCommit,
 			expectedStartup: false,
 		},
+
+		// We started after the DLP was triggered, and try to force
+		// close. This is rejected as we can't force close with local
+		// data loss. We should still be able to start up however.
+		{
+			name:            "ignore force close local data loss",
+			forceCloseErr:   lnwallet.ErrForceCloseLocalDataLoss,
+			expectedState:   StateBroadcastCommit,
+			expectedStartup: true,
+		},
 	}
 
 	for _, test := range tests {
@@ -2741,9 +2778,21 @@ func TestChannelArbitratorStartAfterCommitmentRejected(t *testing.T) {
 				newStates: make(chan ArbitratorState, 5),
 				state:     StateBroadcastCommit,
 			}
-			chanArbCtx, err := createTestChannelArbitrator(t, log)
-			require.NoError(t, err, "unable to create "+
-				"ChannelArbitrator")
+
+			var testOpts []testChanArbOption
+			if test.forceCloseErr != nil {
+				testOpts = append(
+					testOpts,
+					withForceCloseErr(test.forceCloseErr),
+				)
+			}
+
+			chanArbCtx, err := createTestChannelArbitrator(
+				t, log, testOpts...,
+			)
+			require.NoError(
+				t, err, "unable to create ChannelArbitrator",
+			)
 
 			chanArb := chanArbCtx.chanArb
 
@@ -2753,20 +2802,32 @@ func TestChannelArbitratorStartAfterCommitmentRejected(t *testing.T) {
 
 				return test.broadcastErr
 			}
+
 			err = chanArb.Start(nil)
+
 			if !test.expectedStartup {
 				require.ErrorIs(t, err, test.broadcastErr)
 				return
 			}
+
 			require.NoError(t, err)
 
 			t.Cleanup(func() {
 				require.NoError(t, chanArb.Stop())
 			})
 
 			// In case the startup succeeds we check that the state
-			// is as expected.
-			chanArbCtx.AssertStateTransitions(test.expectedState)
+			// is as expected, we only check this if we didn't need
+			// to advance from StateBroadcastCommit.
+			if test.expectedState != StateBroadcastCommit {
+				chanArbCtx.AssertStateTransitions(
+					test.expectedState,
+				)
+			} else {
+				// Otherwise, we expect the state to stay the
+				// same.
+				chanArbCtx.AssertState(test.expectedState)
+			}
 		})
 	}
 }
@@ -2800,6 +2861,8 @@ func assertResolverReport(t *testing.T, reports chan *channeldb.ResolverReport,
 
 type mockChannel struct {
 	anchorResolutions *lnwallet.AnchorResolutions
+
+	forceCloseErr error
 }
 
 func (m *mockChannel) NewAnchorResolutions() (*lnwallet.AnchorResolutions,
@@ -2813,6 +2876,10 @@ func (m *mockChannel) NewAnchorResolutions() (*lnwallet.AnchorResolutions,
 }
 
 func (m *mockChannel) ForceCloseChan() (*lnwallet.LocalForceCloseSummary, error) {
+	if m.forceCloseErr != nil {
+		return nil, m.forceCloseErr
+	}
+
 	summary := &lnwallet.LocalForceCloseSummary{
 		CloseTx:         &wire.MsgTx{},
 		HtlcResolutions: &lnwallet.HtlcResolutions{},

contractcourt/htlc_timeout_resolver.go

@@ -462,6 +462,7 @@ func (h *htlcTimeoutResolver) sweepSecondLevelTx() error {
 			Fee: sweep.FeePreference{
 				ConfTarget: secondLevelConfTarget,
 			},
+			Force: true,
 		},
 	)
 	if err != nil {

docs/release-notes/release-notes-0.17.0.md

@@ -71,12 +71,21 @@ fails](https://github.com/lightningnetwork/lnd/pull/7876).
   retried](https://github.com/lightningnetwork/lnd/pull/7927) with an
   exponential back off.
 
+
 * In the watchtower client, we [now explicitly 
   handle](https://github.com/lightningnetwork/lnd/pull/7981) the scenario where 
   a channel is closed while we still have an in-memory update for it. 
 
+* `lnd` [now properly handles a case where an erroneous force close attempt
+  would impeded start up](https://github.com/lightningnetwork/lnd/pull/7985).
+
 # New Features
 ## Functional Enhancements
+
+* `lnd` can now optionally generate [blocking and mutex
+  profiles](https://github.com/lightningnetwork/lnd/pull/7983). These profiles
+  are useful to attempt to debug high mutex contention, or deadlock scenarios.
+
 ### Protocol Features
 * This release marks the first release that includes the new [musig2-based
   taproot channel type](https://github.com/lightningnetwork/lnd/pull/7904). As
@@ -228,6 +237,11 @@ None
 
 * The [WalletBalance](https://github.com/lightningnetwork/lnd/pull/7857) RPC
   (lncli walletbalance) now supports showing the balance for a specific account.
+
+* The [websockets proxy now uses a larger default max
+  message](https://github.com/lightningnetwork/lnd/pull/7991) size to support
+  proxying larger messages.
+
 
 ## lncli Updates
 * Added ability to use [environment variables to override `lncli` global
@@ -244,6 +258,13 @@ None
 * Add [`--unused`](https://github.com/lightningnetwork/lnd/pull/6387) to
   `lncli newaddr` command.
 
+* [The `MuSig2SessionRequest` proto message now contains a field to allow a
+  caller to specify a custom signing
+  nonce](https://github.com/lightningnetwork/lnd/pull/7994). This can be useful
+  for protocol where an external nonces must be pre-generated before the full
+  session can be completed.
+
+
 ## Code Health
 * Updated [our fork for serializing protobuf as JSON to be based on the
   latest version of `google.golang.org/protobuf` instead of the deprecated

go.mod

@@ -3,9 +3,9 @@ module github.com/lightningnetwork/lnd
 require (
 	github.com/NebulousLabs/go-upnp v0.0.0-20180202185039-29b680b06c82
 	github.com/Yawning/aez v0.0.0-20211027044916-e49e68abd344
-	github.com/btcsuite/btcd v0.23.5-0.20230711222809-7faa9b266231
+	github.com/btcsuite/btcd v0.23.5-0.20230905170901-80f5a0ffdf36
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2
-	github.com/btcsuite/btcd/btcutil v1.1.3
+	github.com/btcsuite/btcd/btcutil v1.1.4-0.20230904040416-d4f519f5dc05
 	github.com/btcsuite/btcd/btcutil/psbt v1.1.8
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
@@ -36,14 +36,14 @@ require (
 	github.com/lightninglabs/neutrino v0.16.0
 	github.com/lightninglabs/neutrino/cache v1.1.1
 	github.com/lightningnetwork/lightning-onion v1.2.1-0.20230823005744-06182b1d7d2f
-	github.com/lightningnetwork/lnd/cert v1.2.1
-	github.com/lightningnetwork/lnd/clock v1.1.0
-	github.com/lightningnetwork/lnd/healthcheck v1.2.2
-	github.com/lightningnetwork/lnd/kvdb v1.4.2
-	github.com/lightningnetwork/lnd/queue v1.1.0
-	github.com/lightningnetwork/lnd/ticker v1.1.0
-	github.com/lightningnetwork/lnd/tlv v1.1.0
-	github.com/lightningnetwork/lnd/tor v1.1.1
+	github.com/lightningnetwork/lnd/cert v1.2.2
+	github.com/lightningnetwork/lnd/clock v1.1.1
+	github.com/lightningnetwork/lnd/healthcheck v1.2.3
+	github.com/lightningnetwork/lnd/kvdb v1.4.4
+	github.com/lightningnetwork/lnd/queue v1.1.1
+	github.com/lightningnetwork/lnd/ticker v1.1.1
+	github.com/lightningnetwork/lnd/tlv v1.1.1
+	github.com/lightningnetwork/lnd/tor v1.1.2
 	github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796
 	github.com/miekg/dns v1.1.43
 	github.com/ory/dockertest/v3 v3.10.0
@@ -121,7 +121,6 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
-	github.com/kr/pretty v0.3.0 // indirect
 	github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
@@ -182,7 +181,6 @@ require (
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/errgo.v1 v1.0.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect