From 84136f91f3c37620c9407ce899466bdc010a6be0 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Wed, 23 Oct 2024 09:20:23 +0200 Subject: [PATCH 1/3] multi: update btclog, lnd and lndclient deps Update the deps so that structured logging is availabel in aperture. --- .github/workflows/main.yml | 7 +----- aperture.go | 11 +++++++--- aperturedb/log.go | 2 +- auth/log.go | 2 +- challenger/log.go | 2 +- config.go | 12 +++++++++- go.mod | 25 ++++++++++++--------- go.sum | 45 ++++++++++++++++++++------------------ hashmail_server_test.go | 8 ++++--- internal/test/log.go | 6 ++--- l402/log.go | 2 +- lnc/log.go | 2 +- log.go | 9 ++++---- proxy/log.go | 2 +- sample-conf.yaml | 9 ++++++++ 15 files changed, 87 insertions(+), 57 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 884657d..4dd63ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,12 +19,7 @@ env: # If you change this value, please change it in the following files as well: # /Dockerfile - # - # Don't bump this until go 1.19 is out (which should include a fix for - # https://github.com/golang/go/issues/51799). There was a race condition - # introduced with go 1.16.10 that causes the unit tests to fail (could also - # happen in production). - GO_VERSION: 1.22.3 + GO_VERSION: 1.22.6 jobs: ######################## diff --git a/aperture.go b/aperture.go index 57eaa71..56b4343 100644 --- a/aperture.go +++ b/aperture.go @@ -618,7 +618,10 @@ func setupLogging(cfg *Config, interceptor signal.Interceptor) error { } // Now initialize the logger and set the log level. - SetupLoggers(logWriter, interceptor) + sugLogMgr = build.NewSubLoggerManager( + build.NewDefaultLogHandlers(cfg.Logging, logWriter)..., + ) + SetupLoggers(sugLogMgr, interceptor) // Use our default data dir unless a base dir is set. logFile := filepath.Join(apertureDataDir, defaultLogFilename) @@ -627,12 +630,14 @@ func setupLogging(cfg *Config, interceptor signal.Interceptor) error { } err := logWriter.InitLogRotator( - logFile, defaultMaxLogFileSize, defaultMaxLogFiles, + logFile, cfg.LogCompressor, defaultMaxLogFileSize, + defaultMaxLogFiles, ) if err != nil { return err } - return build.ParseAndSetDebugLevels(cfg.DebugLevel, logWriter) + + return build.ParseAndSetDebugLevels(cfg.DebugLevel, sugLogMgr) } // getTLSConfig returns a TLS configuration for either a self-signed certificate diff --git a/aperturedb/log.go b/aperturedb/log.go index b786138..a69c0c5 100644 --- a/aperturedb/log.go +++ b/aperturedb/log.go @@ -1,7 +1,7 @@ package aperturedb import ( - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" ) // Subsystem defines the logging code for this subsystem. diff --git a/auth/log.go b/auth/log.go index 457a691..8f8a155 100644 --- a/auth/log.go +++ b/auth/log.go @@ -1,7 +1,7 @@ package auth import ( - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" "github.com/lightningnetwork/lnd/build" ) diff --git a/challenger/log.go b/challenger/log.go index 99033f0..f4488b5 100644 --- a/challenger/log.go +++ b/challenger/log.go @@ -1,7 +1,7 @@ package challenger import ( - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" "github.com/lightningnetwork/lnd/build" ) diff --git a/config.go b/config.go index 7925233..6db5b5b 100644 --- a/config.go +++ b/config.go @@ -9,6 +9,7 @@ import ( "github.com/btcsuite/btcd/btcutil" "github.com/lightninglabs/aperture/aperturedb" "github.com/lightninglabs/aperture/proxy" + "github.com/lightningnetwork/lnd/build" ) var ( @@ -219,6 +220,13 @@ type Config struct { // WriteTimeout is the maximum amount of time to wait for a response to // be fully written. WriteTimeout time.Duration `long:"writetimeout" description:"The maximum amount of time to wait for a response to be fully written."` + + // Logging controls various aspects of aperture logging. + Logging *build.LogConfig `group:"logging" namespace:"logging"` + + // LogCompressor sets the compression algorithm to use when rotating + // logs. + LogCompressor string `long:"logcompressor" description:"Compression algorithm to use when rotating logs." choice:"gzip" choice:"zstd"` } func (c *Config) validate() error { @@ -235,7 +243,7 @@ func (c *Config) validate() error { return nil } -// DefaultConfig returns the default configuration for a sqlite backend. +// DefaultSqliteConfig returns the default configuration for a sqlite backend. func DefaultSqliteConfig() *aperturedb.SqliteConfig { return &aperturedb.SqliteConfig{ SkipMigrations: false, @@ -257,5 +265,7 @@ func NewConfig() *Config { IdleTimeout: defaultIdleTimeout, ReadTimeout: defaultReadTimeout, WriteTimeout: defaultWriteTimeout, + Logging: build.DefaultLogConfig(), + LogCompressor: build.Gzip, } } diff --git a/go.mod b/go.mod index 2dc7eab..01d1145 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.4 github.com/btcsuite/btcd/btcutil v1.1.5 github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 - github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f - github.com/btcsuite/btcwallet/wtxmgr v1.5.3 + github.com/btcsuite/btclog/v2 v2.0.0-20241017175713-3428138b75c7 + github.com/btcsuite/btcwallet/wtxmgr v1.5.4 github.com/fortytw2/leaktest v1.3.0 github.com/golang-migrate/migrate/v4 v4.17.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -22,7 +22,7 @@ require ( github.com/lightninglabs/lightning-node-connect v0.2.5-alpha github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.2 github.com/lightninglabs/lndclient v0.18.4-0 - github.com/lightningnetwork/lnd v0.18.3-beta.rc3.0.20241011124628-ca3bde901eb8 + github.com/lightningnetwork/lnd v0.18.0-beta.rc4.0.20241022180056-b4ef22bf4722 github.com/lightningnetwork/lnd/cert v1.2.2 github.com/lightningnetwork/lnd/clock v1.1.1 github.com/lightningnetwork/lnd/tlv v1.2.6 @@ -56,11 +56,12 @@ require ( github.com/aead/siphash v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd/btcutil/psbt v1.1.8 // indirect - github.com/btcsuite/btcwallet v0.16.10-0.20240809133323-7d3434c65ae2 // indirect - github.com/btcsuite/btcwallet/wallet/txauthor v1.3.4 // indirect - github.com/btcsuite/btcwallet/wallet/txrules v1.2.1 // indirect - github.com/btcsuite/btcwallet/wallet/txsizes v1.2.4 // indirect - github.com/btcsuite/btcwallet/walletdb v1.4.2 // indirect + github.com/btcsuite/btclog v0.0.0-20241003133417-09c4e92e319c // indirect + github.com/btcsuite/btcwallet v0.16.10-0.20240912233857-ffb143c77cc5 // indirect + github.com/btcsuite/btcwallet/wallet/txauthor v1.3.5 // indirect + github.com/btcsuite/btcwallet/wallet/txrules v1.2.2 // indirect + github.com/btcsuite/btcwallet/wallet/txsizes v1.2.5 // indirect + github.com/btcsuite/btcwallet/walletdb v1.4.4 // indirect github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect github.com/btcsuite/winsvc v1.0.0 // indirect @@ -120,7 +121,7 @@ require ( github.com/lightninglabs/neutrino v0.16.1-0.20240425105051-602843d34ffd // indirect github.com/lightninglabs/neutrino/cache v1.1.2 // indirect github.com/lightningnetwork/lightning-onion v1.2.1-0.20240712235311-98bd56499dfb // indirect - github.com/lightningnetwork/lnd/fn v1.2.1 // indirect + github.com/lightningnetwork/lnd/fn v1.2.3 // indirect github.com/lightningnetwork/lnd/healthcheck v1.2.5 // indirect github.com/lightningnetwork/lnd/kvdb v1.4.10 // indirect github.com/lightningnetwork/lnd/queue v1.1.1 // indirect @@ -160,7 +161,7 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect - go.etcd.io/bbolt v1.3.7 // indirect + go.etcd.io/bbolt v1.3.11 // indirect go.etcd.io/etcd/api/v3 v3.5.7 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect go.etcd.io/etcd/client/v2 v2.305.7 // indirect @@ -203,3 +204,7 @@ require ( // We want to format raw bytes as hex instead of base64. The forked version // allows us to specify that as an option. replace google.golang.org/protobuf => github.com/lightninglabs/protobuf-go-hex-display v1.30.0-hex-display + +replace github.com/lightninglabs/lndclient => github.com/ellemouton/lndclient v1.0.1-0.20241023065244-170dbf90b579 + +replace github.com/lightningnetwork/lnd => github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20241026121832-a03d5fcb9e11 diff --git a/go.sum b/go.sum index 2e31c32..5bffc99 100644 --- a/go.sum +++ b/go.sum @@ -87,21 +87,24 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtyd github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btclog v0.0.0-20241003133417-09c4e92e319c h1:4HxD1lBUGUddhzgaNgrCPsFWd7cGYNpeFUgd9ZIgyM0= +github.com/btcsuite/btclog v0.0.0-20241003133417-09c4e92e319c/go.mod h1:w7xnGOhwT3lmrS4H3b/D1XAXxvh+tbhUm8xeHN2y3TQ= +github.com/btcsuite/btclog/v2 v2.0.0-20241017175713-3428138b75c7 h1:3Ct3zN3VCEKVm5nceWBBEKczc+jvTfVyOEG71ob2Yuc= +github.com/btcsuite/btclog/v2 v2.0.0-20241017175713-3428138b75c7/go.mod h1:XItGUfVOxotJL8kkuk2Hj3EVow5KCugXl3wWfQ6K0AE= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcwallet v0.16.10-0.20240809133323-7d3434c65ae2 h1:qa4Avm7p97JroZZyMJADbEb9u853pjleJYSeitENvLc= -github.com/btcsuite/btcwallet v0.16.10-0.20240809133323-7d3434c65ae2/go.mod h1:X2xDre+j1QphTRo54y2TikUzeSvreL1t1aMXrD8Kc5A= -github.com/btcsuite/btcwallet/wallet/txauthor v1.3.4 h1:poyHFf7+5+RdxNp5r2T6IBRD7RyraUsYARYbp/7t4D8= -github.com/btcsuite/btcwallet/wallet/txauthor v1.3.4/go.mod h1:GETGDQuyq+VFfH1S/+/7slLM/9aNa4l7P4ejX6dJfb0= -github.com/btcsuite/btcwallet/wallet/txrules v1.2.1 h1:UZo7YRzdHbwhK7Rhv3PO9bXgTxiOH45edK5qdsdiatk= -github.com/btcsuite/btcwallet/wallet/txrules v1.2.1/go.mod h1:MVSqRkju/IGxImXYPfBkG65FgEZYA4fXchheILMVl8g= -github.com/btcsuite/btcwallet/wallet/txsizes v1.2.4 h1:nmcKAVTv/cmYrs0A4hbiC6Qw+WTLYy/14SmTt3mLnCo= -github.com/btcsuite/btcwallet/wallet/txsizes v1.2.4/go.mod h1:YqJR8WAAHiKIPesZTr9Cx9Az4fRhRLcJ6GcxzRUZCAc= -github.com/btcsuite/btcwallet/walletdb v1.4.2 h1:zwZZ+zaHo4mK+FAN6KeK85S3oOm+92x2avsHvFAhVBE= -github.com/btcsuite/btcwallet/walletdb v1.4.2/go.mod h1:7ZQ+BvOEre90YT7eSq8bLoxTsgXidUzA/mqbRS114CQ= -github.com/btcsuite/btcwallet/wtxmgr v1.5.3 h1:QrWCio9Leh3DwkWfp+A1SURj8pYn3JuTLv3waP5uEro= -github.com/btcsuite/btcwallet/wtxmgr v1.5.3/go.mod h1:M4nQpxGTXiDlSOODKXboXX7NFthmiBNjzAKKNS7Fhjg= +github.com/btcsuite/btcwallet v0.16.10-0.20240912233857-ffb143c77cc5 h1:zYy233eUBvkF3lq2MUkybEhxhDsrRDSgiToIKN57mtk= +github.com/btcsuite/btcwallet v0.16.10-0.20240912233857-ffb143c77cc5/go.mod h1:1HJXYbjJzgumlnxOC2+ViR1U+gnHWoOn7WeK5OfY1eU= +github.com/btcsuite/btcwallet/wallet/txauthor v1.3.5 h1:Rr0njWI3r341nhSPesKQ2JF+ugDSzdPoeckS75SeDZk= +github.com/btcsuite/btcwallet/wallet/txauthor v1.3.5/go.mod h1:+tXJ3Ym0nlQc/iHSwW1qzjmPs3ev+UVWMbGgfV1OZqU= +github.com/btcsuite/btcwallet/wallet/txrules v1.2.2 h1:YEO+Lx1ZJJAtdRrjuhXjWrYsmAk26wLTlNzxt2q0lhk= +github.com/btcsuite/btcwallet/wallet/txrules v1.2.2/go.mod h1:4v+grppsDpVn91SJv+mZT7B8hEV4nSmpREM4I8Uohws= +github.com/btcsuite/btcwallet/wallet/txsizes v1.2.5 h1:93o5Xz9dYepBP4RMFUc9RGIFXwqP2volSWRkYJFrNtI= +github.com/btcsuite/btcwallet/wallet/txsizes v1.2.5/go.mod h1:lQ+e9HxZ85QP7r3kdxItkiMSloSLg1PEGis5o5CXUQw= +github.com/btcsuite/btcwallet/walletdb v1.4.4 h1:BDel6iT/ltYSIYKs0YbjwnEDi7xR3yzABIsQxN2F1L8= +github.com/btcsuite/btcwallet/walletdb v1.4.4/go.mod h1:jk/hvpLFINF0C1kfTn0bfx2GbnFT+Nvnj6eblZALfjs= +github.com/btcsuite/btcwallet/wtxmgr v1.5.4 h1:hJjHy1h/dJwSfD9uDsCwcH21D1iOrus6OrI5gR9E/O0= +github.com/btcsuite/btcwallet/wtxmgr v1.5.4/go.mod h1:lAv0b1Vj9Ig5U8QFm0yiJ9WqPl8yGO/6l7JxdHY1PKE= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/golangcrypto v0.0.0-20150304025918-53f62d9b43e8/go.mod h1:tYvUd8KLhm/oXvUeSEs2VlLghFjQt9+ZaF9ghH0JNjc= @@ -180,6 +183,10 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20241026121832-a03d5fcb9e11 h1:WaKXIKKKVulx4xlIepGH+EJalMqq+khDWkQgdXpxrqk= +github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20241026121832-a03d5fcb9e11/go.mod h1:AAijN90Q69V+vPQ7rSaf7Ui7KGdLBU7g7XKnQb2apw8= +github.com/ellemouton/lndclient v1.0.1-0.20241023065244-170dbf90b579 h1:VBfkVqSHpr7d0ZCX8kmSlPKb/yfwCfmlqPP5eQmndfc= +github.com/ellemouton/lndclient v1.0.1-0.20241023065244-170dbf90b579/go.mod h1:YBHDD/95qTbfpmKJMwSEDhJT9b3lMx8SDzwWetpYQgg= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -474,8 +481,6 @@ github.com/lightninglabs/lightning-node-connect v0.2.5-alpha h1:ZRVChwczFXK0CEbx github.com/lightninglabs/lightning-node-connect v0.2.5-alpha/go.mod h1:A9Pof9fETkH+F67BnOmrBDThPKstqp73wlImWOZvTXQ= github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.2 h1:Er1miPZD2XZwcfE4xoS5AILqP1mj7kqnhbBSxW9BDxY= github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.2/go.mod h1:antQGRDRJiuyQF6l+k6NECCSImgCpwaZapATth2Chv4= -github.com/lightninglabs/lndclient v0.18.4-0 h1:TdorvV9UIw3fjZrNpVKn3fpsOdw2KWF2Eqdx7+++lcY= -github.com/lightninglabs/lndclient v0.18.4-0/go.mod h1:LbINSPfKEdZuTGqqJ+ZmUxXWNvUCaDqrZeJ7/Al0Z3Y= github.com/lightninglabs/neutrino v0.16.1-0.20240425105051-602843d34ffd h1:D8aRocHpoCv43hL8egXEMYyPmyOiefFHZ66338KQB2s= github.com/lightninglabs/neutrino v0.16.1-0.20240425105051-602843d34ffd/go.mod h1:x3OmY2wsA18+Kc3TSV2QpSUewOCiscw2mKpXgZv2kZk= github.com/lightninglabs/neutrino/cache v1.1.2 h1:C9DY/DAPaPxbFC+xNNEI/z1SJY9GS3shmlu5hIQ798g= @@ -484,14 +489,12 @@ github.com/lightninglabs/protobuf-go-hex-display v1.30.0-hex-display h1:pRdza2wl github.com/lightninglabs/protobuf-go-hex-display v1.30.0-hex-display/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= github.com/lightningnetwork/lightning-onion v1.2.1-0.20240712235311-98bd56499dfb h1:yfM05S8DXKhuCBp5qSMZdtSwvJ+GFzl94KbXMNB1JDY= github.com/lightningnetwork/lightning-onion v1.2.1-0.20240712235311-98bd56499dfb/go.mod h1:c0kvRShutpj3l6B9WtTsNTBUtjSmjZXbJd9ZBRQOSKI= -github.com/lightningnetwork/lnd v0.18.3-beta.rc3.0.20241011124628-ca3bde901eb8 h1:+z0s8M0QItH51qMPgFGlRvi6uBltbURQj6u1srTyRb4= -github.com/lightningnetwork/lnd v0.18.3-beta.rc3.0.20241011124628-ca3bde901eb8/go.mod h1:gzVQkOCZxTLzlUPqnI6t68FVGLbiO6Jj+TcLb4b78n0= github.com/lightningnetwork/lnd/cert v1.2.2 h1:71YK6hogeJtxSxw2teq3eGeuy4rHGKcFf0d0Uy4qBjI= github.com/lightningnetwork/lnd/cert v1.2.2/go.mod h1:jQmFn/Ez4zhDgq2hnYSw8r35bqGVxViXhX6Cd7HXM6U= github.com/lightningnetwork/lnd/clock v1.1.1 h1:OfR3/zcJd2RhH0RU+zX/77c0ZiOnIMsDIBjgjWdZgA0= github.com/lightningnetwork/lnd/clock v1.1.1/go.mod h1:mGnAhPyjYZQJmebS7aevElXKTFDuO+uNFFfMXK1W8xQ= -github.com/lightningnetwork/lnd/fn v1.2.1 h1:pPsVGrwi9QBwdLJzaEGK33wmiVKOxs/zc8H7+MamFf0= -github.com/lightningnetwork/lnd/fn v1.2.1/go.mod h1:SyFohpVrARPKH3XVAJZlXdVe+IwMYc4OMAvrDY32kw0= +github.com/lightningnetwork/lnd/fn v1.2.3 h1:Q1OrgNSgQynVheBNa16CsKVov1JI5N2AR6G07x9Mles= +github.com/lightningnetwork/lnd/fn v1.2.3/go.mod h1:SyFohpVrARPKH3XVAJZlXdVe+IwMYc4OMAvrDY32kw0= github.com/lightningnetwork/lnd/healthcheck v1.2.5 h1:aTJy5xeBpcWgRtW/PGBDe+LMQEmNm/HQewlQx2jt7OA= github.com/lightningnetwork/lnd/healthcheck v1.2.5/go.mod h1:G7Tst2tVvWo7cx6mSBEToQC5L1XOGxzZTPB29g9Rv2I= github.com/lightningnetwork/lnd/kvdb v1.4.10 h1:vK89IVv1oVH9ubQWU+EmoCQFeVRaC8kfmOrqHbY5zoY= @@ -661,8 +664,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec h1:FpfFs4EhNehiVfzQttTuxanPIT43FtkkCFypIod8LHo= gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= diff --git a/hashmail_server_test.go b/hashmail_server_test.go index 8ea1507..2f0c109 100644 --- a/hashmail_server_test.go +++ b/hashmail_server_test.go @@ -7,9 +7,11 @@ import ( "math" "net" "net/http" + "os" "testing" "time" + "github.com/btcsuite/btclog/v2" "github.com/lightninglabs/lightning-node-connect/hashmailrpc" "github.com/lightningnetwork/lnd/build" "github.com/lightningnetwork/lnd/lntest/wait" @@ -32,9 +34,9 @@ var ( ) func init() { - logWriter := build.NewRotatingLogWriter() - SetupLoggers(logWriter, signal.Interceptor{}) - _ = build.ParseAndSetDebugLevels("trace,PRXY=warn", logWriter) + logMgr := build.NewSubLoggerManager(btclog.NewDefaultHandler(os.Stdout)) + SetupLoggers(logMgr, signal.Interceptor{}) + _ = build.ParseAndSetDebugLevels("trace,PRXY=warn", logMgr) } func TestHashMailServerReturnStream(t *testing.T) { diff --git a/internal/test/log.go b/internal/test/log.go index e351129..8499d41 100644 --- a/internal/test/log.go +++ b/internal/test/log.go @@ -3,15 +3,15 @@ package test import ( "os" - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" ) // log is a logger that is initialized with no output filters. This // means the package will not perform any logging by default until the caller // requests it. var ( - backendLog = btclog.NewBackend(logWriter{}) - logger = backendLog.Logger("TEST") + backendLog = btclog.NewDefaultHandler(logWriter{}).SubSystem("TEST") + logger = btclog.NewSLogger(backendLog) ) // logWriter implements an io.Writer that outputs to both standard output and diff --git a/l402/log.go b/l402/log.go index 1eab231..74120e9 100644 --- a/l402/log.go +++ b/l402/log.go @@ -1,7 +1,7 @@ package l402 import ( - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" "github.com/lightningnetwork/lnd/build" ) diff --git a/lnc/log.go b/lnc/log.go index 3f5ad9e..a33ddfa 100644 --- a/lnc/log.go +++ b/lnc/log.go @@ -1,7 +1,7 @@ package lnc import ( - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" "github.com/lightningnetwork/lnd/build" ) diff --git a/log.go b/log.go index 62534ce..1c9ce4c 100644 --- a/log.go +++ b/log.go @@ -1,7 +1,7 @@ package aperture import ( - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" "github.com/lightninglabs/aperture/auth" "github.com/lightninglabs/aperture/l402" "github.com/lightninglabs/aperture/proxy" @@ -15,14 +15,15 @@ const Subsystem = "APER" var ( logWriter = build.NewRotatingLogWriter() + sugLogMgr *build.SubLoggerManager log = build.NewSubLogger(Subsystem, nil) ) // SetupLoggers initializes all package-global logger variables. -func SetupLoggers(root *build.RotatingLogWriter, intercept signal.Interceptor) { +func SetupLoggers(root *build.SubLoggerManager, intercept signal.Interceptor) { genLogger := genSubLogger(root, intercept) - logWriter = root + sugLogMgr = root log = build.NewSubLogger(Subsystem, genLogger) lnd.SetSubLogger(root, Subsystem, log) @@ -34,7 +35,7 @@ func SetupLoggers(root *build.RotatingLogWriter, intercept signal.Interceptor) { // genSubLogger creates a logger for a subsystem. We provide an instance of // a signal.Interceptor to be able to shutdown in the case of a critical error. -func genSubLogger(root *build.RotatingLogWriter, +func genSubLogger(root *build.SubLoggerManager, interceptor signal.Interceptor) func(string) btclog.Logger { // Create a shutdown function which will request shutdown from our diff --git a/proxy/log.go b/proxy/log.go index 81d2bad..1f6480b 100644 --- a/proxy/log.go +++ b/proxy/log.go @@ -4,7 +4,7 @@ import ( "fmt" "net" - "github.com/btcsuite/btclog" + "github.com/btcsuite/btclog/v2" "github.com/lightningnetwork/lnd/build" ) diff --git a/sample-conf.yaml b/sample-conf.yaml index 0d09660..0785ddf 100644 --- a/sample-conf.yaml +++ b/sample-conf.yaml @@ -215,3 +215,12 @@ hashmail: prometheus: enabled: true listenaddr: "localhost:9000" + +# Console and file logger settings. +logging: + console: + disable: false + call-site: short + file: + disable: true + call-site: long From 2b53dd868b20d22fb73ef3d790228dd9ebfa5747 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Wed, 23 Oct 2024 09:34:27 +0200 Subject: [PATCH 2/3] makefile: apply tags to install command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fafda69..2d025dc 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ build: install: @$(call print, "Installing aperture.") - $(GOINSTALL) $(PKG)/cmd/aperture + $(GOINSTALL) -tags="${tags}" $(PKG)/cmd/aperture docker-tools: @$(call print, "Building tools docker image.") From 8cd28634e3bc5beebaebc8a84b8561167733d572 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Wed, 11 Sep 2024 17:13:23 +0200 Subject: [PATCH 3/3] hashmail: update to user structured logs This will make querying by stream IDs more uniform (and hence, way easier). --- hashmail_server.go | 115 +++++++++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 52 deletions(-) diff --git a/hashmail_server.go b/hashmail_server.go index 5f14ff0..f7e7b74 100644 --- a/hashmail_server.go +++ b/hashmail_server.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/btcsuite/btclog/v2" "github.com/lightninglabs/lightning-node-connect/hashmailrpc" "github.com/lightningnetwork/lnd/tlv" "github.com/prometheus/client_golang/prometheus" @@ -104,8 +105,8 @@ func (r *readStream) ReadNextMsg(ctx context.Context) ([]byte, error) { // ReturnStream gives up the read stream by passing it back up through the // payment stream. -func (r *readStream) ReturnStream() { - log.Debugf("Returning read stream %x", r.parentStream.id[:]) +func (r *readStream) ReturnStream(ctx context.Context) { + log.DebugS(ctx, "Returning read stream") r.parentStream.ReturnReadStream(r) } @@ -193,7 +194,7 @@ type stream struct { } // newStream creates a new stream independent of any given stream ID. -func newStream(id streamID, limiter *rate.Limiter, +func newStream(ctx context.Context, id streamID, limiter *rate.Limiter, equivAuth func(auth *hashmailrpc.CipherBoxAuth) error, onStale func() error, staleTimeout time.Duration) *stream { @@ -210,7 +211,7 @@ func newStream(id streamID, limiter *rate.Limiter, id: id, equivAuth: equivAuth, limiter: limiter, - status: newStreamStatus(onStale, staleTimeout), + status: newStreamStatus(ctx, onStale, staleTimeout), readBytesChan: make(chan []byte), readErrChan: make(chan error, 1), quit: make(chan struct{}), @@ -305,8 +306,8 @@ func (s *stream) ReturnWriteStream(w *writeStream) { // RequestReadStream attempts to request the read stream from the main backing // stream. If we're unable to obtain it before the timeout, then an error is // returned. -func (s *stream) RequestReadStream() (*readStream, error) { - log.Tracef("HashMailStream(%x): requesting read stream", s.id[:]) +func (s *stream) RequestReadStream(ctx context.Context) (*readStream, error) { + log.TraceS(ctx, "Requested read stream") select { case r := <-s.readStreamChan: @@ -320,8 +321,8 @@ func (s *stream) RequestReadStream() (*readStream, error) { // RequestWriteStream attempts to request the read stream from the main backing // stream. If we're unable to obtain it before the timeout, then an error is // returned. -func (s *stream) RequestWriteStream() (*writeStream, error) { - log.Tracef("HashMailStream(%x): requesting write stream", s.id[:]) +func (s *stream) RequestWriteStream(ctx context.Context) (*writeStream, error) { + log.TraceS(ctx, "Requesting write stream") select { case w := <-s.writeStreamChan: @@ -389,8 +390,10 @@ func (h *hashMailServer) Stop() { } // tearDownStaleStream can be used to tear down a stale mailbox stream. -func (h *hashMailServer) tearDownStaleStream(id streamID) error { - log.Debugf("Tearing down stale HashMail stream: id=%x", id) +func (h *hashMailServer) tearDownStaleStream(ctx context.Context, + id streamID) error { + + log.DebugS(ctx, "Tearing down stale HashMail stream") h.Lock() defer h.Unlock() @@ -428,7 +431,7 @@ func (h *hashMailServer) ValidateStreamAuth(ctx context.Context, } // InitStream attempts to initialize a new stream given a valid descriptor. -func (h *hashMailServer) InitStream( +func (h *hashMailServer) InitStream(ctx context.Context, init *hashmailrpc.CipherBoxAuth) (*hashmailrpc.CipherInitResp, error) { h.Lock() @@ -436,7 +439,7 @@ func (h *hashMailServer) InitStream( streamID := newStreamID(init.Desc.StreamId) - log.Debugf("Creating new HashMail Stream: %x", streamID) + log.DebugS(ctx, "Creating new HashMail Stream") // The stream is already active, and we only allow a single session for // a given stream to exist. @@ -452,10 +455,11 @@ func (h *hashMailServer) InitStream( rate.Every(h.cfg.msgRate), h.cfg.msgBurstAllowance, ) freshStream := newStream( - streamID, limiter, func(auth *hashmailrpc.CipherBoxAuth) error { + ctx, streamID, limiter, + func(auth *hashmailrpc.CipherBoxAuth) error { return nil }, func() error { - return h.tearDownStaleStream(streamID) + return h.tearDownStaleStream(ctx, streamID) }, h.cfg.staleTimeout, ) @@ -470,7 +474,9 @@ func (h *hashMailServer) InitStream( // LookUpReadStream attempts to loop up a new stream. If the stream is found, then // the stream is marked as being active. Otherwise, an error is returned. -func (h *hashMailServer) LookUpReadStream(streamID []byte) (*readStream, error) { +func (h *hashMailServer) LookUpReadStream(ctx context.Context, + streamID []byte) (*readStream, error) { + h.RLock() defer h.RUnlock() @@ -479,12 +485,13 @@ func (h *hashMailServer) LookUpReadStream(streamID []byte) (*readStream, error) return nil, fmt.Errorf("stream not found") } - return stream.RequestReadStream() + return stream.RequestReadStream(ctx) } // LookUpWriteStream attempts to loop up a new stream. If the stream is found, // then the stream is marked as being active. Otherwise, an error is returned. -func (h *hashMailServer) LookUpWriteStream(streamID []byte) (*writeStream, error) { +func (h *hashMailServer) LookUpWriteStream(ctx context.Context, + streamID []byte) (*writeStream, error) { h.RLock() defer h.RUnlock() @@ -494,7 +501,7 @@ func (h *hashMailServer) LookUpWriteStream(streamID []byte) (*writeStream, error return nil, fmt.Errorf("stream not found") } - return stream.RequestWriteStream() + return stream.RequestWriteStream(ctx) } // TearDownStream attempts to tear down a stream which renders both sides of @@ -523,8 +530,7 @@ func (h *hashMailServer) TearDownStream(ctx context.Context, streamID []byte, return err } - log.Debugf("Tearing down HashMail stream: id=%x, auth=%v", - auth.Desc.StreamId, auth.Auth) + log.DebugS(ctx, "Tearing down HashMail stream", "auth", auth.Auth) // At this point we know the auth was valid, so we'll tear down the // stream. @@ -568,16 +574,17 @@ func (h *hashMailServer) NewCipherBox(ctx context.Context, return nil, err } - log.Debugf("New HashMail stream init: id=%x, auth=%v", - init.Desc.StreamId, init.Auth) + ctxl := btclog.WithCtx(ctx, btclog.Hex("stream_id", init.Desc.StreamId)) - if err := h.ValidateStreamAuth(ctx, init); err != nil { - log.Debugf("Stream creation validation failed (id=%x): %v", - init.Desc.StreamId, err) + log.DebugS(ctxl, "New HashMail stream init", "auth", init.Auth) + + if err := h.ValidateStreamAuth(ctxl, init); err != nil { + log.DebugS(ctxl, "Stream creation validation failed", + "err", err) return nil, err } - resp, err := h.InitStream(init) + resp, err := h.InitStream(ctxl, init) if err != nil { return nil, err } @@ -597,8 +604,9 @@ func (h *hashMailServer) DelCipherBox(ctx context.Context, return nil, err } - log.Debugf("New HashMail stream deletion: id=%x, auth=%v", - auth.Desc.StreamId, auth.Auth) + ctxl := btclog.WithCtx(ctx, btclog.Hex("stream_id", auth.Desc.StreamId)) + + log.DebugS(ctxl, "New HashMail stream deletion", "auth", auth.Auth) if err := h.TearDownStream(ctx, auth.Desc.StreamId, auth); err != nil { return nil, err @@ -610,7 +618,7 @@ func (h *hashMailServer) DelCipherBox(ctx context.Context, // SendStream implements the client streaming call to utilize the write end of // a stream to send a message to the read end. func (h *hashMailServer) SendStream(readStream hashmailrpc.HashMail_SendStreamServer) error { - log.Debugf("New HashMail write stream pending...") + log.Debug("New HashMail write stream pending...") // We'll need to receive the first message in order to determine if // this stream exists or not @@ -621,6 +629,9 @@ func (h *hashMailServer) SendStream(readStream hashmailrpc.HashMail_SendStreamSe return err } + ctx := btclog.WithCtx(readStream.Context(), + btclog.Hex("stream_id", cipherBox.Desc.StreamId)) + switch { case cipherBox.Desc == nil: return fmt.Errorf("cipher box descriptor required") @@ -629,12 +640,11 @@ func (h *hashMailServer) SendStream(readStream hashmailrpc.HashMail_SendStreamSe return fmt.Errorf("stream_id required") } - log.Debugf("New HashMail write stream: id=%x", - cipherBox.Desc.StreamId) + log.DebugS(ctx, "New HashMail write stream") // Now that we have the first message, we can attempt to look up the // given stream. - writeStream, err := h.LookUpWriteStream(cipherBox.Desc.StreamId) + writeStream, err := h.LookUpWriteStream(ctx, cipherBox.Desc.StreamId) if err != nil { return err } @@ -643,13 +653,12 @@ func (h *hashMailServer) SendStream(readStream hashmailrpc.HashMail_SendStreamSe // write inactive if the client hangs up on their end. defer writeStream.ReturnStream() - log.Tracef("Sending msg_len=%v to stream_id=%x", len(cipherBox.Msg), - cipherBox.Desc.StreamId) + log.TraceS(ctx, "Sending message to stream", + "msg_len", len(cipherBox.Msg)) // We'll send the first message into the stream, then enter our loop // below to continue to read from the stream and send it to the read // end. - ctx := readStream.Context() if err := writeStream.WriteMsg(ctx, cipherBox.Msg); err != nil { return err } @@ -659,7 +668,7 @@ func (h *hashMailServer) SendStream(readStream hashmailrpc.HashMail_SendStreamSe // exit before shutting down. select { case <-ctx.Done(): - log.Debugf("SendStream: Context done, exiting") + log.DebugS(ctx, "SendStream: Context done, exiting") return nil case <-h.quit: return fmt.Errorf("server shutting down") @@ -669,13 +678,13 @@ func (h *hashMailServer) SendStream(readStream hashmailrpc.HashMail_SendStreamSe cipherBox, err := readStream.Recv() if err != nil { - log.Debugf("SendStream: Exiting write stream RPC "+ - "stream read: %v", err) + log.DebugS(ctx, "SendStream: Exiting write stream RPC "+ + "stream read", err) return err } - log.Tracef("Sending msg_len=%v to stream_id=%x", - len(cipherBox.Msg), cipherBox.Desc.StreamId) + log.TraceS(ctx, "Sending message to stream", + "msg_len", len(cipherBox.Msg)) if err := writeStream.WriteMsg(ctx, cipherBox.Msg); err != nil { return err @@ -689,25 +698,28 @@ func (h *hashMailServer) SendStream(readStream hashmailrpc.HashMail_SendStreamSe func (h *hashMailServer) RecvStream(desc *hashmailrpc.CipherBoxDesc, reader hashmailrpc.HashMail_RecvStreamServer) error { + ctx := btclog.WithCtx(reader.Context(), + btclog.Hex("stream_id", desc.StreamId)) + // First, we'll attempt to locate the stream. We allow any single // entity that knows of the full stream ID to access the read end. - readStream, err := h.LookUpReadStream(desc.StreamId) + readStream, err := h.LookUpReadStream(ctx, desc.StreamId) if err != nil { return err } - log.Debugf("New HashMail read stream: id=%x", desc.StreamId) + log.DebugS(ctx, "New HashMail read stream") // If the reader hangs up, then we'll mark the stream as inactive so // another can take its place. - defer readStream.ReturnStream() + defer readStream.ReturnStream(ctx) for { // Check to see if the stream has been closed or if we need to - // exit before shutting down. + // exit before shutting d[own. select { case <-reader.Context().Done(): - log.Debugf("Read stream context done.") + log.DebugS(ctx, "Read stream context done.") return nil case <-h.quit: return fmt.Errorf("server shutting down") @@ -717,12 +729,11 @@ func (h *hashMailServer) RecvStream(desc *hashmailrpc.CipherBoxDesc, nextMsg, err := readStream.ReadNextMsg(reader.Context()) if err != nil { - log.Debugf("Got error an read stream read: %v", err) + log.ErrorS(ctx, "Got error on read stream read", err) return err } - log.Tracef("Read %v bytes for HashMail stream_id=%x", - len(nextMsg), desc.StreamId) + log.TraceS(ctx, "Read bytes", "msg_len", len(nextMsg)) // In order not to duplicate metric data, we only record this // read if its streamID is odd. We use the base stream ID as the @@ -742,8 +753,8 @@ func (h *hashMailServer) RecvStream(desc *hashmailrpc.CipherBoxDesc, Msg: nextMsg, }) if err != nil { - log.Debugf("Got error when sending on read stream: %v", - err) + log.DebugS(ctx, "Got error when sending on read stream", + "err", err) return err } } @@ -767,7 +778,7 @@ type streamStatus struct { } // newStreamStatus constructs a new streamStatus instance. -func newStreamStatus(onStale func() error, +func newStreamStatus(ctx context.Context, onStale func() error, staleTimeout time.Duration) *streamStatus { if staleTimeout < 0 { @@ -778,7 +789,7 @@ func newStreamStatus(onStale func() error, staleTimer := time.AfterFunc(staleTimeout, func() { if err := onStale(); err != nil { - log.Errorf("error in onStale callback: %v", err) + log.ErrorS(ctx, "Error from onStale callback", err) } })