From 50d0a697a9d76fed9d7ec420bfee38f7646ae1ea Mon Sep 17 00:00:00 2001 From: "Xiaochao Dong (@damnever)" Date: Fri, 15 Nov 2024 16:16:27 +0800 Subject: [PATCH] Ruler: Add support for per-user external labels Signed-off-by: Xiaochao Dong (@damnever) --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 3 + pkg/cortex/modules.go | 7 ++- pkg/cortex/runtime_config.go | 12 +++- pkg/cortex/runtime_config_test.go | 10 +-- pkg/ruler/compat.go | 1 + pkg/ruler/external_labels.go | 68 +++++++++++++++++++++ pkg/ruler/manager.go | 38 ++++++++++-- pkg/ruler/manager_test.go | 14 +++-- pkg/ruler/ruler_test.go | 9 ++- pkg/util/validation/limits.go | 21 +++++++ pkg/util/validation/limits_test.go | 7 ++- 12 files changed, 171 insertions(+), 20 deletions(-) create mode 100644 pkg/ruler/external_labels.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ebdbdce8737..6e7e7e2512b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245 * [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249 * [FEATURE] Distributor: Accept multiple HA Tracker pairs in the same request. #6256 +* [FEATURE] Ruler: Add support for per-user external labels #6340 * [ENHANCEMENT] Query Frontend/Querier: Add an experimental flag `-querier.enable-promql-experimental-functions` to enable experimental promQL functions. #6355 * [ENHANCEMENT] OTLP: Add `-distributor.otlp-max-recv-msg-size` flag to limit OTLP request size in bytes. #6333 * [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 67ae897e024..f3c4552c8c9 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3614,6 +3614,9 @@ query_rejection: # list of rule groups to disable [disabled_rule_groups: | default = []] + +# external labels for alerting rules +[external_labels: | default = []] ``` ### `memberlist_config` diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 222d321168d..e8d71321dcc 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -154,7 +154,8 @@ func (t *Cortex) initRuntimeConfig() (services.Service, error) { // no need to initialize module if load path is empty return nil, nil } - t.Cfg.RuntimeConfig.Loader = loadRuntimeConfig + runtimeConfigLoader := runtimeConfigLoader{cfg: t.Cfg} + t.Cfg.RuntimeConfig.Loader = runtimeConfigLoader.load // make sure to set default limits before we start loading configuration into memory validation.SetDefaultLimitsForYAMLUnmarshalling(t.Cfg.LimitsConfig) @@ -612,14 +613,14 @@ func (t *Cortex) initRuler() (serv services.Service, err error) { } managerFactory := ruler.DefaultTenantManagerFactory(t.Cfg.Ruler, t.Cfg.ExternalPusher, t.Cfg.ExternalQueryable, queryEngine, t.Overrides, metrics, prometheus.DefaultRegisterer) - manager, err = ruler.NewDefaultMultiTenantManager(t.Cfg.Ruler, managerFactory, metrics, prometheus.DefaultRegisterer, util_log.Logger) + manager, err = ruler.NewDefaultMultiTenantManager(t.Cfg.Ruler, t.Overrides, managerFactory, metrics, prometheus.DefaultRegisterer, util_log.Logger) } else { rulerRegisterer := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "ruler"}, prometheus.DefaultRegisterer) // TODO: Consider wrapping logger to differentiate from querier module logger queryable, _, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, t.StoreQueryables, rulerRegisterer, util_log.Logger) managerFactory := ruler.DefaultTenantManagerFactory(t.Cfg.Ruler, t.Distributor, queryable, engine, t.Overrides, metrics, prometheus.DefaultRegisterer) - manager, err = ruler.NewDefaultMultiTenantManager(t.Cfg.Ruler, managerFactory, metrics, prometheus.DefaultRegisterer, util_log.Logger) + manager, err = ruler.NewDefaultMultiTenantManager(t.Cfg.Ruler, t.Overrides, managerFactory, metrics, prometheus.DefaultRegisterer, util_log.Logger) } if err != nil { diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go index 3d612c7cc88..554fa708056 100644 --- a/pkg/cortex/runtime_config.go +++ b/pkg/cortex/runtime_config.go @@ -58,7 +58,11 @@ func (l *runtimeConfigTenantLimits) AllByUserID() map[string]*validation.Limits return nil } -func loadRuntimeConfig(r io.Reader) (interface{}, error) { +type runtimeConfigLoader struct { + cfg Config +} + +func (l runtimeConfigLoader) load(r io.Reader) (interface{}, error) { var overrides = &RuntimeConfigValues{} decoder := yaml.NewDecoder(r) @@ -74,6 +78,12 @@ func loadRuntimeConfig(r io.Reader) (interface{}, error) { return nil, errMultipleDocuments } + for _, ul := range overrides.TenantLimits { + if err := ul.Validate(l.cfg.Distributor.ShardByAllLabels); err != nil { + return nil, err + } + } + return overrides, nil } diff --git a/pkg/cortex/runtime_config_test.go b/pkg/cortex/runtime_config_test.go index e9a7da0448d..544e5696cb4 100644 --- a/pkg/cortex/runtime_config_test.go +++ b/pkg/cortex/runtime_config_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -28,7 +29,8 @@ overrides: '1235': *id001 '1236': *id001 `) - runtimeCfg, err := loadRuntimeConfig(yamlFile) + loader := runtimeConfigLoader{cfg: Config{Distributor: distributor.Config{ShardByAllLabels: true}}} + runtimeCfg, err := loader.load(yamlFile) require.NoError(t, err) limits := validation.Limits{ @@ -51,7 +53,7 @@ func TestLoadRuntimeConfig_ShouldLoadEmptyFile(t *testing.T) { yamlFile := strings.NewReader(` # This is an empty YAML. `) - actual, err := loadRuntimeConfig(yamlFile) + actual, err := runtimeConfigLoader{}.load(yamlFile) require.NoError(t, err) assert.Equal(t, &RuntimeConfigValues{}, actual) } @@ -60,7 +62,7 @@ func TestLoadRuntimeConfig_MissingPointerFieldsAreNil(t *testing.T) { yamlFile := strings.NewReader(` # This is an empty YAML. `) - actual, err := loadRuntimeConfig(yamlFile) + actual, err := runtimeConfigLoader{}.load(yamlFile) require.NoError(t, err) actualCfg, ok := actual.(*RuntimeConfigValues) @@ -102,7 +104,7 @@ overrides: } for _, tc := range cases { - actual, err := loadRuntimeConfig(strings.NewReader(tc)) + actual, err := runtimeConfigLoader{}.load(strings.NewReader(tc)) assert.Equal(t, errMultipleDocuments, err) assert.Nil(t, actual) } diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index b370e34d207..a4ef97f271d 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -153,6 +153,7 @@ type RulesLimits interface { RulerMaxRulesPerRuleGroup(userID string) int RulerQueryOffset(userID string) time.Duration DisabledRuleGroups(userID string) validation.DisabledRuleGroups + ExternalLabels(userID string) labels.Labels } // EngineQueryFunc returns a new engine query function validating max queryLength. diff --git a/pkg/ruler/external_labels.go b/pkg/ruler/external_labels.go new file mode 100644 index 00000000000..e725ca776d7 --- /dev/null +++ b/pkg/ruler/external_labels.go @@ -0,0 +1,68 @@ +package ruler + +import ( + "sync" + + "github.com/prometheus/prometheus/model/labels" +) + +// userExternalLabels checks and merges per-user external labels with global external labels. +type userExternalLabels struct { + global labels.Labels + limits RulesLimits + builder *labels.Builder + + mtx sync.Mutex + users map[string]labels.Labels +} + +func newUserExternalLabels(global labels.Labels, limits RulesLimits) *userExternalLabels { + return &userExternalLabels{ + global: global, + limits: limits, + builder: labels.NewBuilder(nil), + + mtx: sync.Mutex{}, + users: map[string]labels.Labels{}, + } +} + +func (e *userExternalLabels) get(userID string) (labels.Labels, bool) { + e.mtx.Lock() + defer e.mtx.Unlock() + lset, ok := e.users[userID] + return lset, ok +} + +func (e *userExternalLabels) update(userID string) (labels.Labels, bool) { + lset := e.limits.ExternalLabels(userID) + + e.mtx.Lock() + defer e.mtx.Unlock() + + e.builder.Reset(e.global) + for _, l := range lset { + e.builder.Set(l.Name, l.Value) + } + lset = e.builder.Labels() + + if !labels.Equal(e.users[userID], lset) { + e.users[userID] = lset + return lset, true + } + return lset, false +} + +func (e *userExternalLabels) remove(user string) { + e.mtx.Lock() + defer e.mtx.Unlock() + delete(e.users, user) +} + +func (e *userExternalLabels) cleanup() { + e.mtx.Lock() + defer e.mtx.Unlock() + for user := range e.users { + delete(e.users, user) + } +} diff --git a/pkg/ruler/manager.go b/pkg/ruler/manager.go index b173e0749c8..4091648a2b9 100644 --- a/pkg/ruler/manager.go +++ b/pkg/ruler/manager.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" "github.com/prometheus/prometheus/notifier" promRules "github.com/prometheus/prometheus/rules" @@ -47,6 +48,9 @@ type DefaultMultiTenantManager struct { notifiers map[string]*rulerNotifier notifiersDiscoveryMetrics map[string]discovery.DiscovererMetrics + // Per-user externalLabels. + userExternalLabels *userExternalLabels + // rules backup rulesBackupManager *rulesBackupManager @@ -62,7 +66,7 @@ type DefaultMultiTenantManager struct { syncRuleMtx sync.Mutex } -func NewDefaultMultiTenantManager(cfg Config, managerFactory ManagerFactory, evalMetrics *RuleEvalMetrics, reg prometheus.Registerer, logger log.Logger) (*DefaultMultiTenantManager, error) { +func NewDefaultMultiTenantManager(cfg Config, limits RulesLimits, managerFactory ManagerFactory, evalMetrics *RuleEvalMetrics, reg prometheus.Registerer, logger log.Logger) (*DefaultMultiTenantManager, error) { ncfg, err := buildNotifierConfig(&cfg) if err != nil { return nil, err @@ -92,6 +96,7 @@ func NewDefaultMultiTenantManager(cfg Config, managerFactory ManagerFactory, eva frontendPool: newFrontendPool(cfg, logger, reg), ruleEvalMetrics: evalMetrics, notifiers: map[string]*rulerNotifier{}, + userExternalLabels: newUserExternalLabels(cfg.ExternalLabels, limits), notifiersDiscoveryMetrics: notifiersDiscoveryMetrics, mapper: newMapper(cfg.RulePath, logger), userManagers: map[string]RulesManager{}, @@ -146,6 +151,7 @@ func (r *DefaultMultiTenantManager) SyncRuleGroups(ctx context.Context, ruleGrou r.removeNotifier(userID) r.mapper.cleanupUser(userID) + r.userExternalLabels.remove(userID) r.lastReloadSuccessful.DeleteLabelValues(userID) r.lastReloadSuccessfulTimestamp.DeleteLabelValues(userID) r.configUpdatesTotal.DeleteLabelValues(userID) @@ -183,12 +189,13 @@ func (r *DefaultMultiTenantManager) BackUpRuleGroups(ctx context.Context, ruleGr func (r *DefaultMultiTenantManager) syncRulesToManager(ctx context.Context, user string, groups rulespb.RuleGroupList) { // Map the files to disk and return the file names to be passed to the users manager if they // have been updated - update, files, err := r.mapper.MapRules(user, groups.Formatted()) + rulesUpdated, files, err := r.mapper.MapRules(user, groups.Formatted()) if err != nil { r.lastReloadSuccessful.WithLabelValues(user).Set(0) level.Error(r.logger).Log("msg", "unable to map rule files", "user", user, "err", err) return } + externalLabels, externalLabelsUpdated := r.userExternalLabels.update(user) existing := true manager := r.getRulesManager(user, ctx) @@ -201,19 +208,26 @@ func (r *DefaultMultiTenantManager) syncRulesToManager(ctx context.Context, user return } - if !existing || update { + if !existing || rulesUpdated || externalLabelsUpdated { level.Debug(r.logger).Log("msg", "updating rules", "user", user) r.configUpdatesTotal.WithLabelValues(user).Inc() - if update && existing { + if rulesUpdated && existing { r.updateRuleCache(user, manager.RuleGroups()) } - err = manager.Update(r.cfg.EvaluationInterval, files, r.cfg.ExternalLabels, r.cfg.ExternalURL.String(), ruleGroupIterationFunc) + err = manager.Update(r.cfg.EvaluationInterval, files, externalLabels, r.cfg.ExternalURL.String(), ruleGroupIterationFunc) r.deleteRuleCache(user) if err != nil { r.lastReloadSuccessful.WithLabelValues(user).Set(0) level.Error(r.logger).Log("msg", "unable to update rule manager", "user", user, "err", err) return } + if externalLabelsUpdated { + if err = r.notifierApplyExternalLabels(user, externalLabels); err != nil { + r.lastReloadSuccessful.WithLabelValues(user).Set(0) + level.Error(r.logger).Log("msg", "unable to update notifier", "user", user, "err", err) + return + } + } r.lastReloadSuccessful.WithLabelValues(user).Set(1) r.lastReloadSuccessfulTimestamp.WithLabelValues(user).SetToCurrentTime() @@ -348,6 +362,19 @@ func (r *DefaultMultiTenantManager) getOrCreateNotifier(userID string, userManag return n.notifier, nil } +func (r *DefaultMultiTenantManager) notifierApplyExternalLabels(userID string, externalLabels labels.Labels) error { + r.notifiersMtx.Lock() + defer r.notifiersMtx.Unlock() + + n, ok := r.notifiers[userID] + if !ok { + return fmt.Errorf("notifier not found") + } + cfg := *r.notifierCfg // Copy it + cfg.GlobalConfig.ExternalLabels = externalLabels + return n.applyConfig(&cfg) +} + func (r *DefaultMultiTenantManager) getCachedRules(userID string) ([]*promRules.Group, bool) { r.ruleCacheMtx.RLock() defer r.ruleCacheMtx.RUnlock() @@ -402,6 +429,7 @@ func (r *DefaultMultiTenantManager) Stop() { // cleanup user rules directories r.mapper.cleanup() + r.userExternalLabels.cleanup() } func (*DefaultMultiTenantManager) ValidateRuleGroup(g rulefmt.RuleGroup) []error { diff --git a/pkg/ruler/manager_test.go b/pkg/ruler/manager_test.go index ab6b783f8ff..e610e393adf 100644 --- a/pkg/ruler/manager_test.go +++ b/pkg/ruler/manager_test.go @@ -29,8 +29,9 @@ func TestSyncRuleGroups(t *testing.T) { } ruleManagerFactory := RuleManagerFactory(nil, waitDurations) + limits := ruleLimits{externalLabels: labels.FromStrings("from", "cortex")} - m, err := NewDefaultMultiTenantManager(Config{RulePath: dir}, ruleManagerFactory, nil, nil, log.NewNopLogger()) + m, err := NewDefaultMultiTenantManager(Config{RulePath: dir}, limits, ruleManagerFactory, nil, nil, log.NewNopLogger()) require.NoError(t, err) const user = "testUser" @@ -61,6 +62,9 @@ func TestSyncRuleGroups(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{user}, users) require.True(t, ok) + lset, ok := m.userExternalLabels.get(user) + require.True(t, ok) + require.Equal(t, limits.externalLabels, lset) } // Passing empty map / nil stops all managers. @@ -79,6 +83,8 @@ func TestSyncRuleGroups(t *testing.T) { require.NoError(t, err) require.Equal(t, []string(nil), users) require.False(t, ok) + _, ok = m.userExternalLabels.get(user) + require.False(t, ok) } // Resync same rules as before. Previously this didn't restart the manager. @@ -154,7 +160,7 @@ func TestSlowRuleGroupSyncDoesNotSlowdownListRules(t *testing.T) { } ruleManagerFactory := RuleManagerFactory(groupsToReturn, waitDurations) - m, err := NewDefaultMultiTenantManager(Config{RulePath: dir}, ruleManagerFactory, nil, prometheus.NewRegistry(), log.NewNopLogger()) + m, err := NewDefaultMultiTenantManager(Config{RulePath: dir}, ruleLimits{}, ruleManagerFactory, nil, prometheus.NewRegistry(), log.NewNopLogger()) require.NoError(t, err) m.SyncRuleGroups(context.Background(), userRules) @@ -217,7 +223,7 @@ func TestSyncRuleGroupsCleanUpPerUserMetrics(t *testing.T) { ruleManagerFactory := RuleManagerFactory(nil, waitDurations) - m, err := NewDefaultMultiTenantManager(Config{RulePath: dir}, ruleManagerFactory, evalMetrics, reg, log.NewNopLogger()) + m, err := NewDefaultMultiTenantManager(Config{RulePath: dir}, ruleLimits{}, ruleManagerFactory, evalMetrics, reg, log.NewNopLogger()) require.NoError(t, err) const user = "testUser" @@ -265,7 +271,7 @@ func TestBackupRules(t *testing.T) { ruleManagerFactory := RuleManagerFactory(nil, waitDurations) config := Config{RulePath: dir} config.Ring.ReplicationFactor = 3 - m, err := NewDefaultMultiTenantManager(config, ruleManagerFactory, evalMetrics, reg, log.NewNopLogger()) + m, err := NewDefaultMultiTenantManager(config, ruleLimits{}, ruleManagerFactory, evalMetrics, reg, log.NewNopLogger()) require.NoError(t, err) const user1 = "testUser" diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 8daf23e7b68..d1a0d0d0784 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -89,6 +89,7 @@ type ruleLimits struct { disabledRuleGroups validation.DisabledRuleGroups maxQueryLength time.Duration queryOffset time.Duration + externalLabels labels.Labels } func (r ruleLimits) RulerTenantShardSize(_ string) int { @@ -113,6 +114,10 @@ func (r ruleLimits) RulerQueryOffset(_ string) time.Duration { return r.queryOffset } +func (r ruleLimits) ExternalLabels(_ string) labels.Labels { + return r.externalLabels +} + func newEmptyQueryable() storage.Queryable { return storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { return emptyQuerier{}, nil @@ -235,7 +240,7 @@ func newManager(t *testing.T, cfg Config) *DefaultMultiTenantManager { engine, queryable, pusher, logger, overrides, reg := testSetup(t, nil) metrics := NewRuleEvalMetrics(cfg, nil) managerFactory := DefaultTenantManagerFactory(cfg, pusher, queryable, engine, overrides, metrics, nil) - manager, err := NewDefaultMultiTenantManager(cfg, managerFactory, metrics, reg, logger) + manager, err := NewDefaultMultiTenantManager(cfg, ruleLimits{}, managerFactory, metrics, reg, logger) require.NoError(t, err) return manager @@ -293,7 +298,7 @@ func buildRuler(t *testing.T, rulerConfig Config, querierTestConfig *querier.Tes engine, queryable, pusher, logger, overrides, reg := testSetup(t, querierTestConfig) metrics := NewRuleEvalMetrics(rulerConfig, reg) managerFactory := DefaultTenantManagerFactory(rulerConfig, pusher, queryable, engine, overrides, metrics, reg) - manager, err := NewDefaultMultiTenantManager(rulerConfig, managerFactory, metrics, reg, log.NewNopLogger()) + manager, err := NewDefaultMultiTenantManager(rulerConfig, ruleLimits{}, managerFactory, metrics, reg, log.NewNopLogger()) require.NoError(t, err) ruler, err := newRuler( diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 11bdf9c4614..f66ebda05e0 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "flag" + "fmt" "math" "regexp" "strings" @@ -26,6 +27,8 @@ var errMaxGlobalSeriesPerUserValidation = errors.New("The ingester.max-global-se var errDuplicateQueryPriorities = errors.New("duplicate entry of priorities found. Make sure they are all unique, including the default priority") var errCompilingQueryPriorityRegex = errors.New("error compiling query priority regex") var errDuplicatePerLabelSetLimit = errors.New("duplicate per labelSet limits found. Make sure they are all unique") +var errInvalidLabelName = errors.New("invalid label name") +var errInvalidLabelValue = errors.New("invalid label value") // Supported values for enum limits const ( @@ -211,6 +214,8 @@ type Limits struct { AlertmanagerMaxAlertsCount int `yaml:"alertmanager_max_alerts_count" json:"alertmanager_max_alerts_count"` AlertmanagerMaxAlertsSizeBytes int `yaml:"alertmanager_max_alerts_size_bytes" json:"alertmanager_max_alerts_size_bytes"` DisabledRuleGroups DisabledRuleGroups `yaml:"disabled_rule_groups" json:"disabled_rule_groups" doc:"nocli|description=list of rule groups to disable"` + + ExternalLabels labels.Labels `yaml:"external_labels" json:"external_labels" doc:"nocli|description=external labels for alerting rules"` } // RegisterFlags adds the flags required to config this to the given FlagSet @@ -310,6 +315,18 @@ func (l *Limits) Validate(shardByAllLabels bool) error { return errMaxGlobalSeriesPerUserValidation } + if err := l.ExternalLabels.Validate(func(l labels.Label) error { + if !model.LabelName(l.Name).IsValid() { + return fmt.Errorf("%w: %q", errInvalidLabelName, l.Name) + } + if !model.LabelValue(l.Value).IsValid() { + return fmt.Errorf("%w: %q", errInvalidLabelValue, l.Value) + } + return nil + }); err != nil { + return err + } + return nil } @@ -948,6 +965,10 @@ func (o *Overrides) DisabledRuleGroups(userID string) DisabledRuleGroups { return DisabledRuleGroups{} } +func (o *Overrides) ExternalLabels(userID string) labels.Labels { + return o.GetOverridesForUser(userID).ExternalLabels +} + // GetOverridesForUser returns the per-tenant limits with overrides. func (o *Overrides) GetOverridesForUser(userID string) *Limits { if o.tenantLimits != nil { diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 997988ada9e..aa0cc90c030 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -62,13 +63,17 @@ func TestLimits_Validate(t *testing.T) { shardByAllLabels: true, expected: nil, }, + "external-labels invalid": { + limits: Limits{ExternalLabels: labels.Labels{{Name: "123dd", Value: "oo"}}}, + expected: errInvalidLabelName, + }, } for testName, testData := range tests { testData := testData t.Run(testName, func(t *testing.T) { - assert.Equal(t, testData.expected, testData.limits.Validate(testData.shardByAllLabels)) + assert.ErrorIs(t, testData.limits.Validate(testData.shardByAllLabels), testData.expected) }) } }