From 86e93a51aa122cdc376ad11d3c67f823d8528b85 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Fri, 20 Sep 2024 22:28:15 +0200 Subject: [PATCH 01/12] filter 0 value counter --- pkg/alertmanager/alertmanager_metrics.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/alertmanager/alertmanager_metrics.go b/pkg/alertmanager/alertmanager_metrics.go index 496b4f7a1a6..9fc85fc2e9b 100644 --- a/pkg/alertmanager/alertmanager_metrics.go +++ b/pkg/alertmanager/alertmanager_metrics.go @@ -337,8 +337,8 @@ func (m *alertmanagerMetrics) Describe(out chan<- *prometheus.Desc) { func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { data := m.regs.BuildMetricFamiliesPerTenant() - data.SendSumOfCountersPerTenant(out, m.alertsReceived, "alertmanager_alerts_received_total") - data.SendSumOfCountersPerTenant(out, m.alertsInvalid, "alertmanager_alerts_invalid_total") + data.SendSumOfCountersPerTenant(out, m.alertsReceived, "alertmanager_alerts_received_total", dskit_metrics.WithSkipZeroValueMetrics) + data.SendSumOfCountersPerTenant(out, m.alertsInvalid, "alertmanager_alerts_invalid_total", dskit_metrics.WithSkipZeroValueMetrics) data.SendSumOfCountersPerTenant(out, m.numNotifications, "alertmanager_notifications_total", dskit_metrics.WithLabels("integration"), dskit_metrics.WithSkipZeroValueMetrics) data.SendSumOfCountersPerTenant(out, m.numFailedNotifications, "alertmanager_notifications_failed_total", dskit_metrics.WithLabels("integration", "reason"), dskit_metrics.WithSkipZeroValueMetrics) @@ -371,10 +371,10 @@ func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { data.SendMaxOfGaugesPerTenant(out, m.configHashValue, "alertmanager_config_hash") - data.SendSumOfCountersPerTenant(out, m.partialMerges, "alertmanager_partial_state_merges_total") - data.SendSumOfCountersPerTenant(out, m.partialMergesFailed, "alertmanager_partial_state_merges_failed_total") - data.SendSumOfCountersPerTenant(out, m.replicationTotal, "alertmanager_state_replication_total") - data.SendSumOfCountersPerTenant(out, m.replicationFailed, "alertmanager_state_replication_failed_total") + data.SendSumOfCountersPerTenant(out, m.partialMerges, "alertmanager_partial_state_merges_total", dskit_metrics.WithSkipZeroValueMetrics) + data.SendSumOfCountersPerTenant(out, m.partialMergesFailed, "alertmanager_partial_state_merges_failed_total", dskit_metrics.WithSkipZeroValueMetrics) + data.SendSumOfCountersPerTenant(out, m.replicationTotal, "alertmanager_state_replication_total", dskit_metrics.WithSkipZeroValueMetrics) + data.SendSumOfCountersPerTenant(out, m.replicationFailed, "alertmanager_state_replication_failed_total", dskit_metrics.WithSkipZeroValueMetrics) data.SendSumOfCounters(out, m.fetchReplicaStateTotal, "alertmanager_state_fetch_replica_state_total") data.SendSumOfCounters(out, m.fetchReplicaStateFailed, "alertmanager_state_fetch_replica_state_failed_total") data.SendSumOfCounters(out, m.initialSyncTotal, "alertmanager_state_initial_sync_total") From fdb3d595f2b1f4e4302d12e00df6b03947163422 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Mon, 23 Sep 2024 11:50:59 +0200 Subject: [PATCH 02/12] add changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e34917267a6..6be1ed7946b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ ### Grafana Mimir +* [CHANGE] Alertmanager: the following metrics are not exported for a given `user` when the metric value is zero: #9359 + * `cortex_alertmanager_alerts_received_total` + * `cortex_alertmanager_alerts_invalid_total` + * `cortex_alertmanager_partial_state_merges_total` + * `cortex_alertmanager_partial_state_merges_failed_total` + * `cortex_alertmanager_state_replication_total` + * `cortex_alertmanager_state_replication_failed_total` * [CHANGE] Update minimal supported version of Go to 1.22. #9134 * [CHANGE] Store-gateway / querier: enable streaming chunks from store-gateways to queriers by default. #6646 * [CHANGE] Querier: honor the start/end time range specified in the read hints when executing a remote read request. #8431 From 4570e358d445df6fdc72016ad837af03059ce347 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Mon, 23 Sep 2024 17:57:25 +0200 Subject: [PATCH 03/12] update dskit --- CHANGELOG.md | 16 +++++++++------- go.mod | 2 +- go.sum | 2 ++ .../grafana/dskit/metrics/tenant_registries.go | 18 +++++++++++------- vendor/modules.txt | 2 +- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6be1ed7946b..a243751bbce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,6 @@ ### Grafana Mimir -* [CHANGE] Alertmanager: the following metrics are not exported for a given `user` when the metric value is zero: #9359 - * `cortex_alertmanager_alerts_received_total` - * `cortex_alertmanager_alerts_invalid_total` - * `cortex_alertmanager_partial_state_merges_total` - * `cortex_alertmanager_partial_state_merges_failed_total` - * `cortex_alertmanager_state_replication_total` - * `cortex_alertmanager_state_replication_failed_total` * [CHANGE] Update minimal supported version of Go to 1.22. #9134 * [CHANGE] Store-gateway / querier: enable streaming chunks from store-gateways to queriers by default. #6646 * [CHANGE] Querier: honor the start/end time range specified in the read hints when executing a remote read request. #8431 @@ -39,6 +32,15 @@ * [CHANGE] Querier: allow wrapping errors with context errors only when the former actually correspond to `context.Canceled` and `context.DeadlineExceeded`. #9175 * [CHANGE] Query-scheduler: Remove the experimental `-query-scheduler.use-multi-algorithm-query-queue` flag. The new multi-algorithm tree queue is always used for the scheduler. #9210 * [CHANGE] Distributor: reject incoming requests until the distributor service has started. #9317 +* [CHANGE] Alertmanager: the following metrics are not exported for a given `user` when the metric value is zero: #9359 + * `cortex_alertmanager_alerts_received_total` + * `cortex_alertmanager_alerts_invalid_total` + * `cortex_alertmanager_partial_state_merges_total` + * `cortex_alertmanager_partial_state_merges_failed_total` + * `cortex_alertmanager_state_replication_total` + * `cortex_alertmanager_state_replication_failed_total` + * `cortex_alertmanager_alerts` + * `cortex_alertmanager_silences` * [FEATURE] Alertmanager: Added `-alertmanager.log-parsing-label-matchers` to control logging when parsing label matchers. This flag is intended to be used with `-alertmanager.utf8-strict-mode-enabled` to validate UTF-8 strict mode is working as intended. The default value is `false`. #9173 * [FEATURE] Alertmanager: Added `-alertmanager.utf8-migration-logging-enabled` to enable logging of tenant configurations that are incompatible with UTF-8 strict mode. The default value is `false`. #9174 * [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #8422 #8430 #8454 #8455 #8360 #8490 #8508 #8577 #8660 #8671 #8677 #8747 #8850 #8872 #8838 #8911 #8909 #8923 #8924 #8925 #8932 #8933 #8934 #8962 #8986 #8993 #8995 #9008 #9017 #9018 #9019 #9120 #9121 #9136 #9139 #9140 #9145 #9191 #9192 #9194 #9196 #9201 #9212 #9225 #9260 #9272 #9277 #9278 #9280 #9281 #9342 #9343 #9371 diff --git a/go.mod b/go.mod index 8b2ce61d9d9..aab4e82e679 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/google/gopacket v1.1.19 github.com/gorilla/mux v1.8.1 - github.com/grafana/dskit v0.0.0-20240920183844-560bb26f205e + github.com/grafana/dskit v0.0.0-20240923130221-1f324b47eaee github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/json-iterator/go v1.1.12 diff --git a/go.sum b/go.sum index 122299f9e8e..279b939f56d 100644 --- a/go.sum +++ b/go.sum @@ -1252,6 +1252,8 @@ github.com/grafana/alerting v0.0.0-20240906191856-cdc634f213ea h1:AGmVRk+9ZmzuiL github.com/grafana/alerting v0.0.0-20240906191856-cdc634f213ea/go.mod h1:GMLi6d09Xqo96fCVUjNk//rcjP5NKEdjOzfWIffD5r4= github.com/grafana/dskit v0.0.0-20240920183844-560bb26f205e h1:srRxyx7T2OWJzenCktxk2bCsh8hnbzRYSGRkZxMbGK4= github.com/grafana/dskit v0.0.0-20240920183844-560bb26f205e/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ= +github.com/grafana/dskit v0.0.0-20240923130221-1f324b47eaee h1:msHhmdD1R967st2JOpLxp7tkUwRc580H/Ae193VyweY= +github.com/grafana/dskit v0.0.0-20240923130221-1f324b47eaee/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ= github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc h1:BW+LjKJDz0So5LI8UZfW5neWeKpSkWqhmGjQFzcFfLM= github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc/go.mod h1:JVmqPBe8A/pZWwRoJW5ZjyALeY5OXMzPl7LrVXOdZAI= github.com/grafana/goautoneg v0.0.0-20240607115440-f335c04c58ce h1:WI1olbgS+sEl77qxEYbmt9TgRUz7iLqmjh8lYPpGlKQ= diff --git a/vendor/github.com/grafana/dskit/metrics/tenant_registries.go b/vendor/github.com/grafana/dskit/metrics/tenant_registries.go index e1dc153d237..c903877aea5 100644 --- a/vendor/github.com/grafana/dskit/metrics/tenant_registries.go +++ b/vendor/github.com/grafana/dskit/metrics/tenant_registries.go @@ -219,25 +219,29 @@ func (d MetricFamiliesPerTenant) SendSumOfGaugesWithLabels(out chan<- prometheus // SendSumOfGaugesPerTenant provides metrics on a per-tenant basis. // This function assumes that `tenant` is the first label on the provided metric Desc. -func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, gauge string) { - d.SendSumOfGaugesPerTenantWithLabels(out, desc, gauge) -} +func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenant(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, options ...MetricOption) { + opts := applyMetricOptions(options...) -// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the -// first label on the provided metric Desc -func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) { for _, tenantEntry := range d { if tenantEntry.tenant == "" { continue } result := singleValueWithLabelsMap{} - tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, labelNames, gaugeValue, result.aggregateFn, false) + tenantEntry.metrics.sumOfSingleValuesWithLabels(metric, opts.labelNames, gaugeValue, result.aggregateFn, opts.skipZeroValueMetrics) result.prependTenantLabelValue(tenantEntry.tenant) result.WriteToMetricChannel(out, desc, prometheus.GaugeValue) } } +// SendSumOfGaugesPerTenantWithLabels provides metrics with the provided label names on a per-tenant basis. This function assumes that `tenant` is the +// first label on the provided metric Desc +// +// Deprecated: use SendSumOfGaugesPerTenant with WithLabels option instead. +func (d MetricFamiliesPerTenant) SendSumOfGaugesPerTenantWithLabels(out chan<- prometheus.Metric, desc *prometheus.Desc, metric string, labelNames ...string) { + d.SendSumOfGaugesPerTenant(out, desc, metric, WithLabels(labelNames...)) +} + func (d MetricFamiliesPerTenant) sumOfSingleValuesWithLabels(metric string, fn func(*dto.Metric) float64, labelNames []string, skipZeroValue bool) singleValueWithLabelsMap { result := singleValueWithLabelsMap{} for _, tenantEntry := range d { diff --git a/vendor/modules.txt b/vendor/modules.txt index c061f8a263c..4a0186534ac 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -611,7 +611,7 @@ github.com/grafana/alerting/receivers/webex github.com/grafana/alerting/receivers/webhook github.com/grafana/alerting/receivers/wecom github.com/grafana/alerting/templates -# github.com/grafana/dskit v0.0.0-20240920183844-560bb26f205e +# github.com/grafana/dskit v0.0.0-20240923130221-1f324b47eaee ## explicit; go 1.21 github.com/grafana/dskit/backoff github.com/grafana/dskit/ballast From 668fa1a32073ad1f3ba53756c662bbef8e737d8f Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Mon, 23 Sep 2024 17:57:47 +0200 Subject: [PATCH 04/12] ignore zero'ed gauges --- pkg/alertmanager/alertmanager_metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/alertmanager/alertmanager_metrics.go b/pkg/alertmanager/alertmanager_metrics.go index 9fc85fc2e9b..73973ac4a97 100644 --- a/pkg/alertmanager/alertmanager_metrics.go +++ b/pkg/alertmanager/alertmanager_metrics.go @@ -346,7 +346,7 @@ func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfCountersPerTenant(out, m.numNotificationRequestsFailedTotal, "alertmanager_notification_requests_failed_total", dskit_metrics.WithLabels("integration"), dskit_metrics.WithSkipZeroValueMetrics) data.SendSumOfCountersPerTenant(out, m.numNotificationSuppressedTotal, "alertmanager_notifications_suppressed_total", dskit_metrics.WithLabels("reason"), dskit_metrics.WithSkipZeroValueMetrics) data.SendSumOfHistograms(out, m.notificationLatencySeconds, "alertmanager_notification_latency_seconds") - data.SendSumOfGaugesPerTenantWithLabels(out, m.markerAlerts, "alertmanager_alerts", "state") + data.SendSumOfGaugesPerTenant(out, m.markerAlerts, "alertmanager_alerts", dskit_metrics.WithLabels("state"), dskit_metrics.WithSkipZeroValueMetrics) data.SendSumOfSummaries(out, m.nflogGCDuration, "alertmanager_nflog_gc_duration_seconds") data.SendSumOfSummaries(out, m.nflogSnapshotDuration, "alertmanager_nflog_snapshot_duration_seconds") @@ -367,7 +367,7 @@ func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfCounters(out, m.silencesQueryErrorsTotal, "alertmanager_silences_query_errors_total") data.SendSumOfHistograms(out, m.silencesQueryDuration, "alertmanager_silences_query_duration_seconds") data.SendSumOfCounters(out, m.silencesPropagatedMessagesTotal, "alertmanager_silences_gossip_messages_propagated_total") - data.SendSumOfGaugesPerTenantWithLabels(out, m.silences, "alertmanager_silences", "state") + data.SendSumOfGaugesPerTenant(out, m.silences, "alertmanager_silences", dskit_metrics.WithLabels("state"), dskit_metrics.WithSkipZeroValueMetrics) data.SendMaxOfGaugesPerTenant(out, m.configHashValue, "alertmanager_config_hash") From 53955ac085a0ceda468d5f1c2b9c7d88c75712ef Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 09:35:26 +0200 Subject: [PATCH 05/12] remove usages of deprecated gauge helper --- pkg/ruler/manager_metrics.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/ruler/manager_metrics.go b/pkg/ruler/manager_metrics.go index 59d6af59977..9f5a3953fea 100644 --- a/pkg/ruler/manager_metrics.go +++ b/pkg/ruler/manager_metrics.go @@ -153,9 +153,9 @@ func (m *ManagerMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfCountersPerTenant(out, m.IterationsScheduled, "prometheus_rule_group_iterations_total", dskit_metrics.WithLabels("rule_group")) data.SendSumOfCountersPerTenant(out, m.EvalTotal, "prometheus_rule_evaluations_total", dskit_metrics.WithLabels("rule_group")) data.SendSumOfCountersPerTenant(out, m.EvalFailures, "prometheus_rule_evaluation_failures_total", dskit_metrics.WithLabels("rule_group")) - data.SendSumOfGaugesPerTenantWithLabels(out, m.GroupInterval, "prometheus_rule_group_interval_seconds", "rule_group") - data.SendSumOfGaugesPerTenantWithLabels(out, m.GroupLastEvalTime, "prometheus_rule_group_last_evaluation_timestamp_seconds", "rule_group") - data.SendSumOfGaugesPerTenantWithLabels(out, m.GroupLastDuration, "prometheus_rule_group_last_duration_seconds", "rule_group") - data.SendSumOfGaugesPerTenantWithLabels(out, m.GroupRules, "prometheus_rule_group_rules", "rule_group") - data.SendSumOfGaugesPerTenantWithLabels(out, m.GroupLastEvalSamples, "prometheus_rule_group_last_evaluation_samples", "rule_group") + data.SendSumOfGaugesPerTenant(out, m.GroupInterval, "prometheus_rule_group_interval_seconds", dskit_metrics.WithLabels("rule_group")) + data.SendSumOfGaugesPerTenant(out, m.GroupLastEvalTime, "prometheus_rule_group_last_evaluation_timestamp_seconds", dskit_metrics.WithLabels("rule_group")) + data.SendSumOfGaugesPerTenant(out, m.GroupLastDuration, "prometheus_rule_group_last_duration_seconds", dskit_metrics.WithLabels("rule_group")) + data.SendSumOfGaugesPerTenant(out, m.GroupRules, "prometheus_rule_group_rules", dskit_metrics.WithLabels("rule_group")) + data.SendSumOfGaugesPerTenant(out, m.GroupLastEvalSamples, "prometheus_rule_group_last_evaluation_samples", dskit_metrics.WithLabels("rule_group")) } From 3510820e41747eadf350bdf92d36f5bc33d05a77 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 09:46:21 +0200 Subject: [PATCH 06/12] fix go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 279b939f56d..ff7fbc5ff97 100644 --- a/go.sum +++ b/go.sum @@ -1250,8 +1250,6 @@ github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc h1:PXZQA2WCxe85T github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= github.com/grafana/alerting v0.0.0-20240906191856-cdc634f213ea h1:AGmVRk+9ZmzuiLJl6hzQE1vBlVz9wbEb2+J52Gui2ys= github.com/grafana/alerting v0.0.0-20240906191856-cdc634f213ea/go.mod h1:GMLi6d09Xqo96fCVUjNk//rcjP5NKEdjOzfWIffD5r4= -github.com/grafana/dskit v0.0.0-20240920183844-560bb26f205e h1:srRxyx7T2OWJzenCktxk2bCsh8hnbzRYSGRkZxMbGK4= -github.com/grafana/dskit v0.0.0-20240920183844-560bb26f205e/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ= github.com/grafana/dskit v0.0.0-20240923130221-1f324b47eaee h1:msHhmdD1R967st2JOpLxp7tkUwRc580H/Ae193VyweY= github.com/grafana/dskit v0.0.0-20240923130221-1f324b47eaee/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ= github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc h1:BW+LjKJDz0So5LI8UZfW5neWeKpSkWqhmGjQFzcFfLM= From 42ec5af7a8260634b39420df037677ef454699d5 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 10:12:09 +0200 Subject: [PATCH 07/12] fix integration tests --- integration/alertmanager_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index 8ef81ee45dc..f175c16d1cf 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -927,8 +927,11 @@ func TestAlertmanagerShardingScaling(t *testing.T) { e2e.Equals(float64(numUsers*expectedReplication)), "cortex_alertmanager_config_last_reload_successful")) + if expectedSilences > 0 { + expectedSilences *= expectedReplication + } require.NoError(t, ams.WaitSumMetrics( - e2e.Equals(float64(expectedSilences*expectedReplication)), + e2e.Equals(float64(expectedSilences)), "cortex_alertmanager_silences")) } From 25cb0a19c8b4795f46761cbe4b90f95fd4fade1a Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 10:28:38 +0200 Subject: [PATCH 08/12] fix integration tests --- integration/alertmanager_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index f175c16d1cf..16b259ea7b2 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -927,12 +927,13 @@ func TestAlertmanagerShardingScaling(t *testing.T) { e2e.Equals(float64(numUsers*expectedReplication)), "cortex_alertmanager_config_last_reload_successful")) + // we stopped exporting zero silences + // https://github.com/grafana/mimir/pull/9359 if expectedSilences > 0 { - expectedSilences *= expectedReplication + require.NoError(t, ams.WaitSumMetrics( + e2e.Equals(float64(expectedSilences*expectedReplication)), + "cortex_alertmanager_silences")) } - require.NoError(t, ams.WaitSumMetrics( - e2e.Equals(float64(expectedSilences)), - "cortex_alertmanager_silences")) } // Start up the first instance and use it to create some silences. From 5f17809de8620315eb496914ecf72eb3a9e6a9fa Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 13:26:16 +0200 Subject: [PATCH 09/12] wait -> skip --- integration/alertmanager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index 16b259ea7b2..938bd2d0286 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -437,7 +437,7 @@ func TestAlertmanagerStoreAPI(t *testing.T) { require.NoError(t, am.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_alertmanager_alerts_received_total"}, e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "user", "user-1")), - e2e.WaitMissingMetrics)) + e2e.SkipMissingMetrics)) err = c.DeleteAlertmanagerConfig(context.Background()) require.NoError(t, err) @@ -566,7 +566,7 @@ func TestAlertmanagerSharding(t *testing.T) { return alertmanagers.WaitSumMetricsWithOptions( e2e.Equals(float64(amount)), []string{"cortex_alertmanager_silences"}, - e2e.WaitMissingMetrics, + e2e.SkipMissingMetrics, e2e.WithLabelMatchers( labels.MustNewMatcher(labels.MatchEqual, "state", state), ), From 296996536bdaff964071573178b496d402903fe1 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 13:41:48 +0200 Subject: [PATCH 10/12] add wait back for one case --- integration/alertmanager_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index 938bd2d0286..813c63360af 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -437,7 +437,7 @@ func TestAlertmanagerStoreAPI(t *testing.T) { require.NoError(t, am.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_alertmanager_alerts_received_total"}, e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "user", "user-1")), - e2e.SkipMissingMetrics)) + e2e.WaitMissingMetrics)) err = c.DeleteAlertmanagerConfig(context.Background()) require.NoError(t, err) From 9d68f4fa36a119f663ca8a5d3e2eb25694e01833 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 13:54:47 +0200 Subject: [PATCH 11/12] use skipmissing instead of checking for 0 --- integration/alertmanager_test.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index 813c63360af..a9436587fe9 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -927,13 +927,10 @@ func TestAlertmanagerShardingScaling(t *testing.T) { e2e.Equals(float64(numUsers*expectedReplication)), "cortex_alertmanager_config_last_reload_successful")) - // we stopped exporting zero silences - // https://github.com/grafana/mimir/pull/9359 - if expectedSilences > 0 { - require.NoError(t, ams.WaitSumMetrics( - e2e.Equals(float64(expectedSilences*expectedReplication)), - "cortex_alertmanager_silences")) - } + require.NoError(t, ams.WaitSumMetrics( + e2e.Equals(float64(expectedSilences*expectedReplication)), + e2e.SkipMissingMetrics, + "cortex_alertmanager_silences")) } // Start up the first instance and use it to create some silences. From 54db10b6015189f07e454412ec68c716f54299a8 Mon Sep 17 00:00:00 2001 From: Tito Lins Date: Tue, 24 Sep 2024 14:01:43 +0200 Subject: [PATCH 12/12] fix --- integration/alertmanager_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index a9436587fe9..f9b95b659a8 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -927,10 +927,10 @@ func TestAlertmanagerShardingScaling(t *testing.T) { e2e.Equals(float64(numUsers*expectedReplication)), "cortex_alertmanager_config_last_reload_successful")) - require.NoError(t, ams.WaitSumMetrics( + require.NoError(t, ams.WaitSumMetricsWithOptions( e2e.Equals(float64(expectedSilences*expectedReplication)), - e2e.SkipMissingMetrics, - "cortex_alertmanager_silences")) + []string{"cortex_alertmanager_silences"}, + e2e.SkipMissingMetrics)) } // Start up the first instance and use it to create some silences.