From 3d35357e93caef4a9e2f2757c6e1543af3b4ea9d Mon Sep 17 00:00:00 2001 From: QuentinBisson Date: Tue, 9 Apr 2024 14:45:37 +0200 Subject: [PATCH 1/3] Ignore prometheus-agent-config-if-mimir-enabled Signed-off-by: QuentinBisson --- CHANGELOG.md | 4 ++++ pkg/prometheusquerier/querier.go | 4 ++-- service/controller/clusterapi/resource.go | 22 ++++++++++++++---- .../controller/managementcluster/resource.go | 23 ++++++++++++++----- .../monitoring/remotewriteconfig/create.go | 4 ++-- .../monitoring/remotewriteconfig/resource.go | 8 +++---- 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69a6bb37d..40e5282c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Remove prometheus remote write agent configuration when mimir is enabled. + ## [4.72.0] - 2024-04-03 ### Changed diff --git a/pkg/prometheusquerier/querier.go b/pkg/prometheusquerier/querier.go index 9ed603d08..ad8970d0b 100644 --- a/pkg/prometheusquerier/querier.go +++ b/pkg/prometheusquerier/querier.go @@ -13,7 +13,7 @@ import ( ) // QueryInstant performs an instant query against a Prometheus server. -func QueryTSDBHeadSeries(cluster string) (float64, error) { +func QueryTSDBHeadSeries(ctx context.Context, cluster string) (float64, error) { config := api.Config{ Address: fmt.Sprintf("http://prometheus-operated.%s-prometheus.svc:9090/%s", cluster, cluster), RoundTripper: http.DefaultTransport, @@ -28,7 +28,7 @@ func QueryTSDBHeadSeries(cluster string) (float64, error) { // Run query against client. api := v1.NewAPI(c) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) val, _, err := api.Query(ctx, "max_over_time(prometheus_tsdb_head_series[6h])", time.Now()) // Ignoring warnings for now. cancel() if err != nil { diff --git a/service/controller/clusterapi/resource.go b/service/controller/clusterapi/resource.go index c32b8660b..ba0651cab 100644 --- a/service/controller/clusterapi/resource.go +++ b/service/controller/clusterapi/resource.go @@ -181,9 +181,14 @@ func New(config Config) ([]resource.Interface, error) { return nil, microerror.Mask(err) } } + organizationReader := organization.NewNamespaceReader(config.K8sClient.K8sClient(), config.Installation, config.Provider) + // This resource creates a the prometheus agent remote write configuration. + // This is now managed by the observability-operator when mimir is enabled. var remoteWriteConfigResource resource.Interface - { + if config.MimirEnabled { + remoteWriteConfigResource = noop.New(noop.Config{Logger: config.Logger}) + } else { c := remotewriteconfig.Config{ K8sClient: config.K8sClient, Logger: config.Logger, @@ -203,8 +208,12 @@ func New(config Config) ([]resource.Interface, error) { } } + // This resource creates a the prometheus agent remote write secret. + // This is now managed by the observability-operator when mimir is enabled. var remoteWriteSecretResource resource.Interface - { + if config.MimirEnabled { + remoteWriteSecretResource = noop.New(noop.Config{Logger: config.Logger}) + } else { c := remotewritesecret.Config{ K8sClient: config.K8sClient, Logger: config.Logger, @@ -221,8 +230,11 @@ func New(config Config) ([]resource.Interface, error) { } } + // This resource is not used in latest observability bundle versions. var remoteWriteAPIEndpointConfigSecretResource resource.Interface - { + if config.MimirEnabled { + remoteWriteAPIEndpointConfigSecretResource = noop.New(noop.Config{Logger: config.Logger}) + } else { c := remotewriteapiendpointconfigsecret.Config{ K8sClient: config.K8sClient, Logger: config.Logger, @@ -390,13 +402,13 @@ func New(config Config) ([]resource.Interface, error) { rbacResource, heartbeatWebhookConfigResource, scrapeConfigResource, + alertmanagerWiringResource, + prometheusResource, remoteWriteConfigResource, remoteWriteSecretResource, remoteWriteAPIEndpointConfigSecretResource, remoteWriteIngressAuthResource, remoteWriteIngressResource, - alertmanagerWiringResource, - prometheusResource, verticalPodAutoScalerResource, ingressResource, heartbeatResource, diff --git a/service/controller/managementcluster/resource.go b/service/controller/managementcluster/resource.go index 0db94b5ad..fa9138fda 100644 --- a/service/controller/managementcluster/resource.go +++ b/service/controller/managementcluster/resource.go @@ -84,6 +84,7 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) { var err error passwordManager := password.SimpleManager{} + organizationReader := organization.NewNamespaceReader(config.K8sClient.K8sClient(), config.Installation, config.Provider) var namespaceResource resource.Interface { @@ -237,7 +238,6 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) { } } - organizationReader := organization.NewNamespaceReader(config.K8sClient.K8sClient(), config.Installation, config.Provider) var scrapeConfigResource resource.Interface { c := scrapeconfigs.Config{ @@ -337,8 +337,12 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) { } } + // This resource creates a the prometheus agent remote write configuration. + // This is now managed by the observability-operator when mimir is enabled. var remoteWriteConfigResource resource.Interface - { + if config.MimirEnabled { + remoteWriteConfigResource = noop.New(noop.Config{Logger: config.Logger}) + } else { c := remotewriteconfig.Config{ K8sClient: config.K8sClient, Logger: config.Logger, @@ -358,8 +362,12 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) { } } + // This resource creates a the prometheus agent remote write secret. + // This is now managed by the observability-operator when mimir is enabled. var remoteWriteSecretResource resource.Interface - { + if config.MimirEnabled { + remoteWriteSecretResource = noop.New(noop.Config{Logger: config.Logger}) + } else { c := remotewritesecret.Config{ K8sClient: config.K8sClient, Logger: config.Logger, @@ -376,8 +384,11 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) { } } + // This resource is not used in latest observability bundle versions. var remoteWriteAPIEndpointConfigSecretResource resource.Interface - { + if config.MimirEnabled { + remoteWriteAPIEndpointConfigSecretResource = noop.New(noop.Config{Logger: config.Logger}) + } else { c := remotewriteapiendpointconfigsecret.Config{ K8sClient: config.K8sClient, Logger: config.Logger, @@ -407,13 +418,13 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) { ciliumnetpolResource, heartbeatWebhookConfigResource, alertmanagerWiringResource, + scrapeConfigResource, + prometheusResource, remoteWriteConfigResource, remoteWriteSecretResource, remoteWriteAPIEndpointConfigSecretResource, remoteWriteIngressAuthResource, remoteWriteIngressResource, - scrapeConfigResource, - prometheusResource, verticalPodAutoScalerResource, monitoringIngressResource, heartbeatResource, diff --git a/service/controller/resource/monitoring/remotewriteconfig/create.go b/service/controller/resource/monitoring/remotewriteconfig/create.go index 5c503de19..5c3d7eb4a 100644 --- a/service/controller/resource/monitoring/remotewriteconfig/create.go +++ b/service/controller/resource/monitoring/remotewriteconfig/create.go @@ -29,7 +29,7 @@ func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error { // Get the current configmap if it exists. current, err := r.k8sClient.K8sClient().CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { - err = r.createConfigMap(ctx, cluster, name, namespace, r.version) + err = r.createConfigMap(ctx, cluster, name, namespace) if err != nil { return microerror.Mask(err) } @@ -48,7 +48,7 @@ func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error { return microerror.Mask(err) } - desired, err := r.desiredConfigMap(ctx, cluster, name, namespace, r.version, shards) + desired, err := r.desiredConfigMap(ctx, cluster, name, namespace, shards) if err != nil { return microerror.Mask(err) } diff --git a/service/controller/resource/monitoring/remotewriteconfig/resource.go b/service/controller/resource/monitoring/remotewriteconfig/resource.go index 4e3acd0bf..18901b690 100644 --- a/service/controller/resource/monitoring/remotewriteconfig/resource.go +++ b/service/controller/resource/monitoring/remotewriteconfig/resource.go @@ -96,7 +96,7 @@ func (r *Resource) Name() string { return Name } -func (r *Resource) desiredConfigMap(ctx context.Context, cluster metav1.Object, name string, namespace string, version string, shards int) (*corev1.ConfigMap, error) { +func (r *Resource) desiredConfigMap(ctx context.Context, cluster metav1.Object, name string, namespace string, shards int) (*corev1.ConfigMap, error) { organization, err := r.organizationReader.Read(ctx, cluster) if err != nil { return nil, microerror.Mask(err) @@ -151,7 +151,7 @@ func (r *Resource) desiredConfigMap(ctx context.Context, cluster metav1.Object, // We want to compute the number of shards based on the number of nodes. func (r *Resource) getShardsCountForCluster(ctx context.Context, cluster metav1.Object, currentShardCount int) (int, error) { - headSeries, err := prometheusquerier.QueryTSDBHeadSeries(key.ClusterID(cluster)) + headSeries, err := prometheusquerier.QueryTSDBHeadSeries(ctx, key.ClusterID(cluster)) if err != nil { // If prometheus is not accessible (for instance, not running because this is a new cluster, we check if prometheus is accessible) var dnsError *net.DNSError @@ -164,13 +164,13 @@ func (r *Resource) getShardsCountForCluster(ctx context.Context, cluster metav1. return computeShards(currentShardCount, headSeries), nil } -func (r *Resource) createConfigMap(ctx context.Context, cluster metav1.Object, name string, namespace string, version string) error { +func (r *Resource) createConfigMap(ctx context.Context, cluster metav1.Object, name string, namespace string) error { shards, err := r.getShardsCountForCluster(ctx, cluster, 1) if err != nil { return microerror.Mask(err) } - configMap, err := r.desiredConfigMap(ctx, cluster, name, namespace, version, shards) + configMap, err := r.desiredConfigMap(ctx, cluster, name, namespace, shards) if err != nil { return microerror.Mask(err) } From 334985862778b4d00431c00b81610663377b85fb Mon Sep 17 00:00:00 2001 From: Quentin Bisson Date: Thu, 16 May 2024 14:07:09 +0200 Subject: [PATCH 2/3] Update querier.go --- pkg/prometheusquerier/querier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/prometheusquerier/querier.go b/pkg/prometheusquerier/querier.go index ad8970d0b..9ed603d08 100644 --- a/pkg/prometheusquerier/querier.go +++ b/pkg/prometheusquerier/querier.go @@ -13,7 +13,7 @@ import ( ) // QueryInstant performs an instant query against a Prometheus server. -func QueryTSDBHeadSeries(ctx context.Context, cluster string) (float64, error) { +func QueryTSDBHeadSeries(cluster string) (float64, error) { config := api.Config{ Address: fmt.Sprintf("http://prometheus-operated.%s-prometheus.svc:9090/%s", cluster, cluster), RoundTripper: http.DefaultTransport, @@ -28,7 +28,7 @@ func QueryTSDBHeadSeries(ctx context.Context, cluster string) (float64, error) { // Run query against client. api := v1.NewAPI(c) - ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) val, _, err := api.Query(ctx, "max_over_time(prometheus_tsdb_head_series[6h])", time.Now()) // Ignoring warnings for now. cancel() if err != nil { From 285aef0a899a46ecbe244a141cf80f69ca550b02 Mon Sep 17 00:00:00 2001 From: QuentinBisson Date: Thu, 16 May 2024 14:13:58 +0200 Subject: [PATCH 3/3] fix build --- pkg/prometheusquerier/querier.go | 4 ++-- .../resource/monitoring/remotewriteconfig/create.go | 2 +- .../resource/monitoring/remotewriteconfig/resource.go | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/prometheusquerier/querier.go b/pkg/prometheusquerier/querier.go index 9ed603d08..ad8970d0b 100644 --- a/pkg/prometheusquerier/querier.go +++ b/pkg/prometheusquerier/querier.go @@ -13,7 +13,7 @@ import ( ) // QueryInstant performs an instant query against a Prometheus server. -func QueryTSDBHeadSeries(cluster string) (float64, error) { +func QueryTSDBHeadSeries(ctx context.Context, cluster string) (float64, error) { config := api.Config{ Address: fmt.Sprintf("http://prometheus-operated.%s-prometheus.svc:9090/%s", cluster, cluster), RoundTripper: http.DefaultTransport, @@ -28,7 +28,7 @@ func QueryTSDBHeadSeries(cluster string) (float64, error) { // Run query against client. api := v1.NewAPI(c) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) val, _, err := api.Query(ctx, "max_over_time(prometheus_tsdb_head_series[6h])", time.Now()) // Ignoring warnings for now. cancel() if err != nil { diff --git a/service/controller/resource/monitoring/remotewriteconfig/create.go b/service/controller/resource/monitoring/remotewriteconfig/create.go index 86e58c92f..5c3d7eb4a 100644 --- a/service/controller/resource/monitoring/remotewriteconfig/create.go +++ b/service/controller/resource/monitoring/remotewriteconfig/create.go @@ -43,7 +43,7 @@ func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error { return microerror.Mask(err) } - shards, err := r.getShardsCountForCluster(cluster, currentShards) + shards, err := r.getShardsCountForCluster(ctx, cluster, currentShards) if err != nil { return microerror.Mask(err) } diff --git a/service/controller/resource/monitoring/remotewriteconfig/resource.go b/service/controller/resource/monitoring/remotewriteconfig/resource.go index afdd3ea32..ea38819be 100644 --- a/service/controller/resource/monitoring/remotewriteconfig/resource.go +++ b/service/controller/resource/monitoring/remotewriteconfig/resource.go @@ -157,14 +157,14 @@ func (r *Resource) desiredConfigMap(ctx context.Context, cluster metav1.Object, } // We want to compute the number of shards based on the number of nodes. -func (r *Resource) getShardsCountForCluster(cluster metav1.Object, currentShardCount int) (int, error) { +func (r *Resource) getShardsCountForCluster(ctx context.Context, cluster metav1.Object, currentShardCount int) (int, error) { clusterShardingStrategy, err := key.GetClusterShardingStrategy(cluster) if err != nil { return 0, microerror.Mask(err) } shardingStrategy := r.shardingStrategy.Merge(clusterShardingStrategy) - headSeries, err := prometheusquerier.QueryTSDBHeadSeries(key.ClusterID(cluster)) + headSeries, err := prometheusquerier.QueryTSDBHeadSeries(ctx, key.ClusterID(cluster)) if err != nil { // If prometheus is not accessible (for instance, not running because this is a new cluster, we check if prometheus is accessible) var dnsError *net.DNSError @@ -177,8 +177,8 @@ func (r *Resource) getShardsCountForCluster(cluster metav1.Object, currentShardC return shardingStrategy.ComputeShards(currentShardCount, headSeries), nil } -func (r *Resource) createConfigMap(ctx context.Context, cluster metav1.Object, name string, namespace string, version string) error { - shards, err := r.getShardsCountForCluster(cluster, 1) +func (r *Resource) createConfigMap(ctx context.Context, cluster metav1.Object, name string, namespace string) error { + shards, err := r.getShardsCountForCluster(ctx, cluster, 1) if err != nil { return microerror.Mask(err) }