diff --git a/cmd/event-reporter-server/commands/event_reporter_server.go b/cmd/event-reporter-server/commands/event_reporter_server.go index 1e91a5d2b5bf1..29a1a3c9b5c8e 100644 --- a/cmd/event-reporter-server/commands/event_reporter_server.go +++ b/cmd/event-reporter-server/commands/event_reporter_server.go @@ -95,10 +95,9 @@ func NewCommand() *cobra.Command { rootpath string useGrpc bool - rateLimiterEnabled bool - rateLimiterBucketSize int - rateLimiterDuration time.Duration - rateLimiterLearningMode bool + rateLimiterEnabled bool + rateLimiterBucketSize int + rateLimiterDuration time.Duration ) var command = &cobra.Command{ Use: cliName, @@ -179,10 +178,9 @@ func NewCommand() *cobra.Command { AuthToken: codefreshToken, }, RateLimiterOpts: &reporter.RateLimiterOpts{ - Enabled: rateLimiterEnabled, - Rate: rateLimiterDuration, - Capacity: rateLimiterBucketSize, - LearningMode: rateLimiterLearningMode, + Enabled: rateLimiterEnabled, + Rate: rateLimiterDuration, + Capacity: rateLimiterBucketSize, }, } @@ -233,7 +231,6 @@ func NewCommand() *cobra.Command { command.Flags().BoolVar(&rateLimiterEnabled, "rate-limiter-enabled", env.ParseBoolFromEnv("RATE_LIMITER_ENABLED", false), "Use rate limiter for prevent queue to be overflowed") command.Flags().IntVar(&rateLimiterBucketSize, "rate-limiter-bucket-size", env.ParseNumFromEnv("RATE_LIMITER_BUCKET_SIZE", math.MaxInt, 0, math.MaxInt), "The maximum amount of requests allowed per window.") command.Flags().DurationVar(&rateLimiterDuration, "rate-limiter-period", env.ParseDurationFromEnv("RATE_LIMITER_DURATION", 24*time.Hour, 0, math.MaxInt64), "The rate limit window size.") - command.Flags().BoolVar(&rateLimiterLearningMode, "rate-limiter-learning-mode", env.ParseBoolFromEnv("RATE_LIMITER_LEARNING_MODE_ENABLED", false), "The rate limit enabled in learning mode ( not blocking sending to queue but logging it )") cacheSrc = servercache.AddCacheFlagsToCmd(command, func(client *redis.Client) { redisClient = client }) diff --git a/event_reporter/metrics/metrics.go b/event_reporter/metrics/metrics.go index b84e4c538e002..0c7861b28589f 100644 --- a/event_reporter/metrics/metrics.go +++ b/event_reporter/metrics/metrics.go @@ -70,7 +70,7 @@ var ( Name: "cf_e_reporter_app_events_size", Help: "Size of specific application events queue of taked shard.", }, - []string{"reporter_shard", "application", "got_in_queue", "error_in_learning_mode"}, + []string{"reporter_shard", "application", "got_in_queue"}, ) erroredEventsCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -145,8 +145,8 @@ func (m *MetricsServer) SetQueueSizeCounter(size int) { m.queueSizeCounter.WithLabelValues(m.shard).Set(float64(size)) } -func (m *MetricsServer) IncAppEventsCounter(application string, gotToProcessingQueue bool, errorInLearningMode bool) { - m.appEventsCounter.WithLabelValues(m.shard, application, strconv.FormatBool(gotToProcessingQueue), strconv.FormatBool(errorInLearningMode)).Inc() +func (m *MetricsServer) IncAppEventsCounter(application string, gotToProcessingQueue bool) { + m.appEventsCounter.WithLabelValues(m.shard, application, strconv.FormatBool(gotToProcessingQueue)).Inc() } func (m *MetricsServer) IncErroredEventsCounter(metricEventType MetricEventType, errorType MetricEventErrorType, application string) { diff --git a/event_reporter/reporter/broadcaster.go b/event_reporter/reporter/broadcaster.go index 10878066e9eb4..4d2b58f3beebf 100644 --- a/event_reporter/reporter/broadcaster.go +++ b/event_reporter/reporter/broadcaster.go @@ -78,27 +78,23 @@ func (b *broadcasterHandler) notify(event *appv1.ApplicationWatchEvent) { for _, s := range subscribers { if s.matches(event) { - duration, err, learningMode := b.rateLimiter.Limit(event.Application.Name) - errorInLearningMode := learningMode && err != nil + duration, err := b.rateLimiter.Limit(event.Application.Name) if err != nil { - log.Errorf("adding application '%s' to channel failed, due to rate limit, duration left %s, learningMode %t", event.Application.Name, duration.String(), learningMode) - // if learning mode is enabled, we will continue to send events - if !learningMode { - b.metricsServer.IncAppEventsCounter(event.Application.Name, false, false) - continue - } + log.Infof("adding application '%s' to channel failed, due to rate limit, duration left %s", event.Application.Name, duration.String()) + b.metricsServer.IncAppEventsCounter(event.Application.Name, false) + continue } select { case s.ch <- event: { log.Infof("adding application '%s' to channel", event.Application.Name) - b.metricsServer.IncAppEventsCounter(event.Application.Name, true, errorInLearningMode) + b.metricsServer.IncAppEventsCounter(event.Application.Name, true) } default: // drop event if cannot send right away log.WithField("application", event.Application.Name).Warn("unable to send event notification") - b.metricsServer.IncAppEventsCounter(event.Application.Name, false, errorInLearningMode) + b.metricsServer.IncAppEventsCounter(event.Application.Name, false) } } } diff --git a/event_reporter/reporter/rate_limiter.go b/event_reporter/reporter/rate_limiter.go index 116f48b469792..3b65f7db41da4 100644 --- a/event_reporter/reporter/rate_limiter.go +++ b/event_reporter/reporter/rate_limiter.go @@ -7,10 +7,9 @@ import ( ) type RateLimiterOpts struct { - Enabled bool - Rate time.Duration - Capacity int - LearningMode bool + Enabled bool + Rate time.Duration + Capacity int } type RateLimiter struct { @@ -22,9 +21,9 @@ func NewRateLimiter(opts *RateLimiterOpts) *RateLimiter { return &RateLimiter{opts: opts, limiters: make(map[string]*limiters.FixedWindow)} } -func (rl *RateLimiter) Limit(applicationName string) (time.Duration, error, bool) { +func (rl *RateLimiter) Limit(applicationName string) (time.Duration, error) { if !rl.opts.Enabled { - return time.Duration(0), nil, rl.opts.LearningMode + return time.Duration(0), nil } limiter := rl.limiters[applicationName] @@ -33,6 +32,5 @@ func (rl *RateLimiter) Limit(applicationName string) (time.Duration, error, bool rl.limiters[applicationName] = limiter } - duration, err := limiter.Limit(context.Background()) - return duration, err, rl.opts.LearningMode + return limiter.Limit(context.Background()) } diff --git a/event_reporter/reporter/rate_limiter_test.go b/event_reporter/reporter/rate_limiter_test.go index 499374f3f4159..bba89a9a39929 100644 --- a/event_reporter/reporter/rate_limiter_test.go +++ b/event_reporter/reporter/rate_limiter_test.go @@ -10,7 +10,7 @@ func TestRateLimiter(t *testing.T) { rl := NewRateLimiter(&RateLimiterOpts{ Enabled: false, }) - d, err, _ := rl.Limit("foo") + d, err := rl.Limit("foo") if err != nil { t.Errorf("Expected no error, got %v", err) } @@ -24,7 +24,7 @@ func TestRateLimiter(t *testing.T) { Rate: time.Second, Capacity: 1, }) - d, err, _ := rl.Limit("foo") + d, err := rl.Limit("foo") if err != nil { t.Errorf("Expected no error, got %v", err) } @@ -38,7 +38,7 @@ func TestRateLimiter(t *testing.T) { Rate: time.Second, Capacity: 0, }) - _, err, _ := rl.Limit("foo") + _, err := rl.Limit("foo") if err == nil { t.Errorf("Expected error, got nil") } diff --git a/go.mod b/go.mod index cbb8227ac98b6..ad8ceafa1a53b 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,6 @@ require ( github.com/ktrysmt/go-bitbucket v0.9.67 github.com/mattn/go-isatty v0.0.19 github.com/mattn/go-zglob v0.0.4 - github.com/mennanov/limiters v1.2.3 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/olekukonko/tablewriter v0.0.5 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -84,7 +83,7 @@ require ( golang.org/x/sync v0.3.0 golang.org/x/term v0.13.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc - google.golang.org/grpc v1.56.3 + google.golang.org/grpc v1.56.2 google.golang.org/protobuf v1.31.0 gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v2 v2.4.0 @@ -112,52 +111,25 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.5.2 // indirect - github.com/armon/go-metrics v0.4.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.17.6 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.17 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.17 // indirect - github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.19.1 // indirect - github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.24 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.8 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.20.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.0 // indirect github.com/aws/smithy-go v1.13.5 // indirect - github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/fatih/color v1.14.1 // indirect - github.com/go-redsync/redsync/v4 v4.8.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/hashicorp/consul/api v1.18.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-hclog v1.4.0 // indirect - github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/serf v0.10.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.7 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect - go.etcd.io/etcd/client/v3 v3.5.7 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - go.uber.org/zap v1.24.0 // indirect google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect gopkg.in/retry.v1 v1.0.3 // indirect diff --git a/go.sum b/go.sum index 4ba70f3c0419b..4ad4665f76ca2 100644 --- a/go.sum +++ b/go.sum @@ -890,6 +890,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -1573,6 +1574,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mennanov/limiters v1.2.3 h1:WmXWn1QWqEfcECt0xFuTnuLZhcmNVfZE6dmTWvIrrpE= github.com/mennanov/limiters v1.2.3/go.mod h1:sfXAtfzVXScTObHoh8sHPQmjEX9uGb4aUiwVdWtsw7Y= +github.com/mennanov/limiters v1.2.4 h1:vjSslzfCsR2XqZWTemTycoz8u21nvlQXs25d5Dv21qo= +github.com/mennanov/limiters v1.2.4/go.mod h1:sfXAtfzVXScTObHoh8sHPQmjEX9uGb4aUiwVdWtsw7Y= github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 h1:YH424zrwLTlyHSH/GzLMJeu5zhYVZSx5RQxGKm1h96s= github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5/go.mod h1:PoGiBqKSQK1vIfQ+yVaFcGjDySHvym6FM1cNYnwzbrY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -1997,6 +2000,7 @@ github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaD go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=