Fix head series query (#74)

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - [Alloy] Enable VPA for AlloyMetrics
+- Change the PromQL query used to determine the amount of head series when scaling Prometheus Agent and Alloy ([#74](https://github.com/giantswarm/observability-operator/pull/74))
 
 ### Fixed
 

pkg/monitoring/alloy/configmap.go

@@ -57,7 +57,7 @@ func (a *Service) GenerateAlloyMonitoringConfigMapData(ctx context.Context, curr
 	}
 
 	// Compute the number of shards based on the number of series.
-	query := fmt.Sprintf(`sum(max_over_time(prometheus_remote_write_wal_storage_active_series{cluster_id="%s", component_id="prometheus.remote_write.default", service="%s"}[6h]))`, cluster.Name, commonmonitoring.AlloyMonitoringAgentAppName)
+	query := fmt.Sprintf(`sum(max_over_time((sum(prometheus_remote_write_wal_storage_active_series{cluster_id="%s", component_id="prometheus.remote_write.default", service="%s"})by(pod))[6h:1h]))`, cluster.Name, commonmonitoring.AlloyMonitoringAgentAppName)
 	headSeries, err := querier.QueryTSDBHeadSeries(ctx, query)
 	if err != nil {
 		logger.Error(err, "alloy-service - failed to query head series")

pkg/monitoring/prometheusagent/configmap.go

@@ -45,7 +45,7 @@ func (pas PrometheusAgentService) buildRemoteWriteConfig(ctx context.Context,
 	}
 
 	// Compute the number of shards based on the number of series.
-	query := fmt.Sprintf("sum(max_over_time(prometheus_agent_active_series{cluster_id=\"%s\"}[6h]))", cluster.Name)
+	query := fmt.Sprintf(`sum(max_over_time((sum(prometheus_agent_active_series{cluster_id="%s"})by(pod))[6h:1h]))`, cluster.Name)
 	headSeries, err := querier.QueryTSDBHeadSeries(ctx, query)
 	if err != nil {
 		logger.Error(err, "failed to query head series")