From 679de064708ceeefc8155285efbc6dfa61ae668a Mon Sep 17 00:00:00 2001 From: narumi Date: Sat, 7 Dec 2024 03:26:35 +0800 Subject: [PATCH 1/2] Add sentinel strategy --- config/sentinel.yaml | 19 ++++ go.mod | 1 + go.sum | 2 + pkg/cmd/strategy/builtin.go | 1 + pkg/datatype/floats/slice.go | 18 ++++ pkg/strategy/sentinel/strategy.go | 150 ++++++++++++++++++++++++++++++ 6 files changed, 191 insertions(+) create mode 100644 config/sentinel.yaml create mode 100644 pkg/strategy/sentinel/strategy.go diff --git a/config/sentinel.yaml b/config/sentinel.yaml new file mode 100644 index 0000000000..5e1aa66b36 --- /dev/null +++ b/config/sentinel.yaml @@ -0,0 +1,19 @@ +sessions: + max: + exchange: &exchange max + envVarPrefix: max + +persistence: + json: + directory: var/data + redis: + host: 127.0.0.1 + port: 6379 + db: 0 + +exchangeStrategies: +- on: *exchange + sentinel: + symbol: BTCUSDT + interval: 1m + scoreThreshold: 0.6 diff --git a/go.mod b/go.mod index 05c860737a..c286fb0791 100644 --- a/go.mod +++ b/go.mod @@ -125,6 +125,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/narumiruna/go-iforest v0.2.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect diff --git a/go.sum b/go.sum index c7356ef8b6..51afd5fffe 100644 --- a/go.sum +++ b/go.sum @@ -446,6 +446,8 @@ github.com/muesli/kmeans v0.3.0/go.mod h1:eNyybq0tX9/iBEP6EMU4Y7dpmGK0uEhODdZpnG github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/narumiruna/go-iforest v0.2.2 h1:48GGRVLSlgtV3vGr+eedXODn5RT3WvYroqpMNEoQvkk= +github.com/narumiruna/go-iforest v0.2.2/go.mod h1:2pumoiqKf0Lr+KvLECMC8uNrbRkxtSvUwMJC/6AW7DM= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= diff --git a/pkg/cmd/strategy/builtin.go b/pkg/cmd/strategy/builtin.go index 8c061ecc31..4fbdfa6379 100644 --- a/pkg/cmd/strategy/builtin.go +++ b/pkg/cmd/strategy/builtin.go @@ -40,6 +40,7 @@ import ( _ "github.com/c9s/bbgo/pkg/strategy/rsmaker" _ "github.com/c9s/bbgo/pkg/strategy/schedule" _ "github.com/c9s/bbgo/pkg/strategy/scmaker" + _ "github.com/c9s/bbgo/pkg/strategy/sentinel" _ "github.com/c9s/bbgo/pkg/strategy/supertrend" _ "github.com/c9s/bbgo/pkg/strategy/support" _ "github.com/c9s/bbgo/pkg/strategy/swing" diff --git a/pkg/datatype/floats/slice.go b/pkg/datatype/floats/slice.go index 1d610a4f53..52f58f5795 100644 --- a/pkg/datatype/floats/slice.go +++ b/pkg/datatype/floats/slice.go @@ -87,6 +87,24 @@ func (s Slice) Mean() (mean float64) { return s.Sum() / float64(length) } +func (s Slice) Var() float64 { + length := len(s) + if length == 0 { + panic("zero length slice") + } + + mean := s.Mean() + variance := 0.0 + for _, v := range s { + variance += (v - mean) * (v - mean) + } + return variance / float64(length) +} + +func (s Slice) Std() float64 { + return math.Sqrt(s.Var()) +} + func (s Slice) Tail(size int) Slice { length := len(s) if length <= size { diff --git a/pkg/strategy/sentinel/strategy.go b/pkg/strategy/sentinel/strategy.go new file mode 100644 index 0000000000..f1230ee447 --- /dev/null +++ b/pkg/strategy/sentinel/strategy.go @@ -0,0 +1,150 @@ +package sentinel + +import ( + "context" + "time" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/datatype/floats" + "github.com/c9s/bbgo/pkg/types" + "github.com/narumiruna/go-iforest/pkg/iforest" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" +) + +const ID = "sentinel" + +func init() { + bbgo.RegisterStrategy(ID, &Strategy{}) +} + +type Strategy struct { + Symbol string `json:"symbol"` + Interval types.Interval `json:"interval"` + ScoreThreshold float64 `json:"scoreThreshold"` + KLineLimit int `json:"klineLimit"` + Window int `json:"window"` + + IsolationForest *iforest.IsolationForest `json:"isolationForest"` + NotificationInterval time.Duration `json:"notificationInterval"` + + notificationRateLimiter *rate.Limiter +} + +func (s *Strategy) ID() string { + return ID +} + +func (s *Strategy) Defaults() error { + if s.ScoreThreshold == 0 { + s.ScoreThreshold = 0.6 + } + + if s.KLineLimit == 0 { + s.KLineLimit = 1440 + } + + if s.Window == 0 { + s.Window = 60 + } + + if s.NotificationInterval == 0 { + s.NotificationInterval = 10 * time.Minute + } + + s.notificationRateLimiter = rate.NewLimiter(rate.Every(s.NotificationInterval), 1) + return nil +} + +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { + session.Subscribe(types.KLineChannel, s.Symbol, types.SubscribeOptions{Interval: s.Interval}) +} + +func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + session.MarketDataStream.OnKLine(types.KLineWith(s.Symbol, s.Interval, func(kline types.KLine) { + if !s.isMarketAvailable(session, s.Symbol) { + return + } + + klines, err := s.queryKLines(ctx, session) + if err != nil { + log.Errorf("Unable to query klines: %v", err) + return + } + + volumes := s.extractVolumes(klines) + samples := s.generateSamples(volumes) + + if s.shouldSkipIsolationForest(volumes, samples) { + s.logSkipIsolationForest(samples, volumes, kline) + return + } + + s.fitIsolationForest(samples) + scores := s.IsolationForest.Score(samples) + s.handleIsolationForestScore(scores, kline) + })) + return nil +} + +func (s *Strategy) isMarketAvailable(session *bbgo.ExchangeSession, symbol string) bool { + _, ok := session.Market(symbol) + return ok +} + +func (s *Strategy) queryKLines(ctx context.Context, session *bbgo.ExchangeSession) ([]types.KLine, error) { + endTime := time.Now() + options := types.KLineQueryOptions{ + Limit: s.KLineLimit, + EndTime: &endTime, + } + return session.Exchange.QueryKLines(ctx, s.Symbol, s.Interval, options) +} + +func (s *Strategy) extractVolumes(klines []types.KLine) floats.Slice { + volumes := floats.Slice{} + for _, kline := range klines { + volumes.Push(kline.Volume.Float64()) + } + return volumes +} + +func (s *Strategy) generateSamples(volumes floats.Slice) [][]float64 { + samples := make([][]float64, 0, len(volumes)) + for i := range volumes { + if i < s.Window { + continue + } + + subset := volumes.Tail(s.Window) + + mean := subset.Mean() + std := subset.Std() + samples = append(samples, []float64{mean, std}) + } + return samples +} + +func (s *Strategy) shouldSkipIsolationForest(volumes floats.Slice, samples [][]float64) bool { + volumeMean := volumes.Mean() + lastMovingMean := samples[len(samples)-1][0] + return lastMovingMean < volumeMean +} + +func (s *Strategy) logSkipIsolationForest(samples [][]float64, volumes floats.Slice, kline types.KLine) { + log.Infof("Skipping isolation forest calculation for symbol: %s, last moving mean: %f, average volume: %f, kline: %s", s.Symbol, samples[len(samples)-1][0], volumes.Mean(), kline.String()) +} + +func (s *Strategy) fitIsolationForest(samples [][]float64) { + s.IsolationForest = iforest.New() + s.IsolationForest.Fit(samples) + log.Infof("Isolation forest fitted with %d samples and %d/%d trees", len(samples), len(s.IsolationForest.Trees), s.IsolationForest.NumTrees) +} + +func (s *Strategy) handleIsolationForestScore(scores []float64, kline types.KLine) { + lastScore := scores[len(scores)-1] + log.Warnf("Symbol: %s, isolation forest score: %f, threshold: %f, kline: %s", s.Symbol, lastScore, s.ScoreThreshold, kline.String()) + if lastScore > s.ScoreThreshold && s.notificationRateLimiter.Allow() { + bbgo.Notify("symbol: %s isolation forest score: %f", s.Symbol, lastScore) + } +} From c8d78a7bca7b40379c98350662f85ec45f1459bd Mon Sep 17 00:00:00 2001 From: narumi Date: Sun, 15 Dec 2024 19:44:57 +0800 Subject: [PATCH 2/2] set retraining rate limiter --- pkg/strategy/sentinel/strategy.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/strategy/sentinel/strategy.go b/pkg/strategy/sentinel/strategy.go index f1230ee447..82c1ba67df 100644 --- a/pkg/strategy/sentinel/strategy.go +++ b/pkg/strategy/sentinel/strategy.go @@ -27,8 +27,10 @@ type Strategy struct { IsolationForest *iforest.IsolationForest `json:"isolationForest"` NotificationInterval time.Duration `json:"notificationInterval"` + RetrainingInterval time.Duration `json:"retrainingInterval"` notificationRateLimiter *rate.Limiter + retrainingRateLimiter *rate.Limiter } func (s *Strategy) ID() string { @@ -52,7 +54,12 @@ func (s *Strategy) Defaults() error { s.NotificationInterval = 10 * time.Minute } + if s.RetrainingInterval == 0 { + s.RetrainingInterval = 1 * time.Hour + } + s.notificationRateLimiter = rate.NewLimiter(rate.Every(s.NotificationInterval), 1) + s.retrainingRateLimiter = rate.NewLimiter(rate.Every(s.RetrainingInterval), 1) return nil } @@ -82,7 +89,7 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se s.fitIsolationForest(samples) scores := s.IsolationForest.Score(samples) - s.handleIsolationForestScore(scores, kline) + s.notifyOnIsolationForestScore(scores, kline) })) return nil } @@ -136,15 +143,19 @@ func (s *Strategy) logSkipIsolationForest(samples [][]float64, volumes floats.Sl } func (s *Strategy) fitIsolationForest(samples [][]float64) { - s.IsolationForest = iforest.New() - s.IsolationForest.Fit(samples) - log.Infof("Isolation forest fitted with %d samples and %d/%d trees", len(samples), len(s.IsolationForest.Trees), s.IsolationForest.NumTrees) + if s.retrainingRateLimiter.Allow() { + s.IsolationForest = iforest.New() + s.IsolationForest.Fit(samples) + log.Infof("Isolation forest fitted with %d samples and %d/%d trees", len(samples), len(s.IsolationForest.Trees), s.IsolationForest.NumTrees) + } } -func (s *Strategy) handleIsolationForestScore(scores []float64, kline types.KLine) { +func (s *Strategy) notifyOnIsolationForestScore(scores []float64, kline types.KLine) { lastScore := scores[len(scores)-1] log.Warnf("Symbol: %s, isolation forest score: %f, threshold: %f, kline: %s", s.Symbol, lastScore, s.ScoreThreshold, kline.String()) - if lastScore > s.ScoreThreshold && s.notificationRateLimiter.Allow() { - bbgo.Notify("symbol: %s isolation forest score: %f", s.Symbol, lastScore) + if lastScore > s.ScoreThreshold { + if s.notificationRateLimiter.Allow() { + bbgo.Notify("symbol: %s isolation forest score: %f", s.Symbol, lastScore) + } } }