From d110ac797668f4af512ef562879bfb3f9f79aa18 Mon Sep 17 00:00:00 2001 From: Kaustav Majumder Date: Tue, 20 Feb 2024 11:35:13 +0530 Subject: [PATCH] Added prometheus support Signed-off-by: Kaustav Majumder --- config/rbac/role.yaml | 35 +++ controllers/defaults/resources.go | 33 +++ .../ocsinitialization_controller.go | 200 ++++++++++++++++++ controllers/platform/platform_detection.go | 23 +- controllers/util/util.go | 27 +++ deploy/csv-templates/ocs-operator.csv.yaml.in | 35 +++ ...s-metrics-servicemonitor-role-binding.yaml | 12 ++ .../k8s-metrics-servicemonitor-role.yaml | 51 +++++ .../ocs-operator.clusterserviceversion.yaml | 35 +++ .../odf-prometheus-role-binding.yaml | 12 ++ .../manifests/odf-prometheus-role.yaml | 24 +++ main.go | 2 + ...s-metrics-servicemonitor-role-binding.yaml | 12 ++ rbac/k8s-metrics-servicemonitor-role.yaml | 51 +++++ rbac/odf-prometheus-role-binding.yaml | 12 ++ rbac/odf-prometheus-role.yaml | 24 +++ templates/alertmanager.go | 14 ++ templates/k8smetricsservicemonitor.go | 70 ++++++ templates/prometheus.go | 92 ++++++++ templates/prometheuskuberbacproxyconfig.go | 36 ++++ templates/prometheusproxynetworkpolicy.go | 42 ++++ 21 files changed, 838 insertions(+), 4 deletions(-) create mode 100644 deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role-binding.yaml create mode 100644 deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role.yaml create mode 100644 deploy/ocs-operator/manifests/odf-prometheus-role-binding.yaml create mode 100644 deploy/ocs-operator/manifests/odf-prometheus-role.yaml create mode 100644 rbac/k8s-metrics-servicemonitor-role-binding.yaml create mode 100644 rbac/k8s-metrics-servicemonitor-role.yaml create mode 100644 rbac/odf-prometheus-role-binding.yaml create mode 100644 rbac/odf-prometheus-role.yaml create mode 100644 templates/alertmanager.go create mode 100644 templates/k8smetricsservicemonitor.go create mode 100644 templates/prometheus.go create mode 100644 templates/prometheuskuberbacproxyconfig.go create mode 100644 templates/prometheusproxynetworkpolicy.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a322815201..bd6216583e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -174,6 +174,19 @@ rules: - get - list - watch +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - monitoring.coreos.com resources: @@ -186,6 +199,28 @@ rules: - list - update - watch +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - get + - list + - update + - watch - apiGroups: - noobaa.io resources: diff --git a/controllers/defaults/resources.go b/controllers/defaults/resources.go index a9031ebe96..1f4226378f 100644 --- a/controllers/defaults/resources.go +++ b/controllers/defaults/resources.go @@ -224,4 +224,37 @@ var ( }, }, } + + MonitoringResources = map[string]corev1.ResourceRequirements{ + "kube-rbac-proxy": { + Requests: corev1.ResourceList{ + "memory": resource.MustParse("30Mi"), + "cpu": resource.MustParse("50m"), + }, + Limits: corev1.ResourceList{ + "memory": resource.MustParse("30Mi"), + "cpu": resource.MustParse("50m"), + }, + }, + "alertmanager": { + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("200Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("200Mi"), + }, + }, + "prometheus": { + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("400m"), + "memory": resource.MustParse("250Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("400m"), + "memory": resource.MustParse("250Mi"), + }, + }, + } ) diff --git a/controllers/ocsinitialization/ocsinitialization_controller.go b/controllers/ocsinitialization/ocsinitialization_controller.go index e969995f5a..e3fe3f8f62 100644 --- a/controllers/ocsinitialization/ocsinitialization_controller.go +++ b/controllers/ocsinitialization/ocsinitialization_controller.go @@ -8,9 +8,13 @@ import ( "github.com/go-logr/logr" secv1client "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1" + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/platform" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" + "github.com/red-hat-storage/ocs-operator/v4/templates" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -56,6 +60,9 @@ type OCSInitializationReconciler struct { // +kubebuilder:rbac:groups=ocs.openshift.io,resources=*,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=get;create;update // +kubebuilder:rbac:groups=security.openshift.io,resourceNames=privileged,resources=securitycontextconstraints,verbs=get;create;update +// +kubebuilder:rbac:groups="networking.k8s.io",resources=networkpolicies,verbs=create;get;list;watch;update +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources={alertmanagers,prometheuses},verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=servicemonitors,verbs=get;list;watch;update;patch;create;delete // Reconcile reads that state of the cluster for a OCSInitialization object and makes changes based on the state read // and what is in the OCSInitialization.Spec @@ -174,6 +181,49 @@ func (r *OCSInitializationReconciler) Reconcile(ctx context.Context, request rec r.Log.Error(err, "Failed to ensure uxbackend service") return reconcile.Result{}, err } + isROSAHCP, err := platform.IsPlatformROSAHCP() + if err != nil { + r.Log.Error(err, "Failed to determine if on ROSA HCP platform") + return reconcile.Result{}, err + } + if isROSAHCP { + r.Log.Info("Setting up monitoring resources for ROSA HCP platform") + err = r.reconcilePrometheusKubeRBACConfigMap(instance) + if err != nil { + r.Log.Error(err, "Failed to ensure kubeRBACConfig config map") + return reconcile.Result{}, err + } + + err = r.reconcilePrometheusService(instance) + if err != nil { + r.Log.Error(err, "Failed to ensure prometheus service") + return reconcile.Result{}, err + } + + err = r.reconcilePrometheus(instance) + if err != nil { + r.Log.Error(err, "Failed to ensure prometheus instance") + return reconcile.Result{}, err + } + + err = r.reconcileAlertManager(instance) + if err != nil { + r.Log.Error(err, "Failed to ensure alertmanager instance") + return reconcile.Result{}, err + } + + err = r.reconcilePrometheusProxyNetworkPolicy(instance) + if err != nil { + r.Log.Error(err, "Failed to ensure Prometheus proxy network policy") + return reconcile.Result{}, err + } + + err = r.reconcileK8sMetricsServiceMonitor(instance) + if err != nil { + r.Log.Error(err, "Failed to ensure k8sMetricsService Monitor") + return reconcile.Result{}, err + } + } reason := ocsv1.ReconcileCompleted message := ocsv1.ReconcileCompletedMessage @@ -192,7 +242,9 @@ func (r *OCSInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&ocsv1.OCSInitialization{}). Owns(&corev1.Service{}). + Owns(&networkingv1.NetworkPolicy{}). Owns(&corev1.Secret{}). + Owns(&promv1.Prometheus{}). // Watcher for storagecluster required to update // ocs-operator-config configmap if storagecluster spec changes Watches( @@ -423,3 +475,151 @@ func (r *OCSInitializationReconciler) reconcileUXBackendService(initialData *ocs return nil } + +func (r *OCSInitializationReconciler) reconcilePrometheusKubeRBACConfigMap(initialData *ocsv1.OCSInitialization) error { + + var err error + prometheusKubeRBACConfigMap := &corev1.ConfigMap{} + prometheusKubeRBACConfigMap.Name = templates.PrometheusKubeRBACProxyConfigMapName + prometheusKubeRBACConfigMap.Namespace = initialData.Namespace + + _, err = ctrl.CreateOrUpdate(r.ctx, r.Client, prometheusKubeRBACConfigMap, func() error { + if err := ctrl.SetControllerReference(initialData, prometheusKubeRBACConfigMap, r.Scheme); err != nil { + return err + } + prometheusKubeRBACConfigMap.Data = templates.KubeRBACProxyConfigMap.Data + return nil + }) + + if err != nil { + r.Log.Error(err, "Failed to create/update prometheus kube-rbac-proxy config map") + return err + } + r.Log.Info("Prometheus kube-rbac-proxy config map creation succeeded", "Name", prometheusKubeRBACConfigMap.Name) + return nil +} + +func (r *OCSInitializationReconciler) reconcilePrometheusService(initialData *ocsv1.OCSInitialization) error { + var err error + prometheusService := &corev1.Service{} + prometheusService.Name = "prometheus" + prometheusService.Namespace = initialData.Namespace + + _, err = ctrl.CreateOrUpdate(r.ctx, r.Client, prometheusService, func() error { + if err := ctrl.SetControllerReference(initialData, prometheusService, r.Scheme); err != nil { + return err + } + util.AddAnnotation(prometheusService, "service.beta.openshift.io/serving-cert-secret-name", "prometheus-serving-cert-secret") + util.AddLabels(prometheusService, "prometheus", "odf-prometheus") + prometheusService.Spec.Selector = map[string]string{ + "app.kubernetes.io/name": prometheusService.Name, + } + prometheusService.Spec.Ports = []corev1.ServicePort{ + { + Name: "https", + Protocol: corev1.ProtocolTCP, + Port: int32(templates.KubeRBACProxyPortNumber), + TargetPort: intstr.FromString("https"), + }, + } + return nil + }) + if err != nil { + r.Log.Error(err, "Failed to create/update prometheus service") + return err + } + r.Log.Info("Service creation succeeded", "Name", prometheusService.Name) + return nil +} + +func (r *OCSInitializationReconciler) reconcilePrometheus(initialData *ocsv1.OCSInitialization) error { + var err error + + prometheus := &promv1.Prometheus{} + prometheus.Name = "odf-prometheus" + prometheus.Namespace = initialData.Namespace + + _, err = ctrl.CreateOrUpdate(r.ctx, r.Client, prometheus, func() error { + if err := ctrl.SetControllerReference(initialData, prometheus, r.Scheme); err != nil { + return err + } + templates.PrometheusTemplate.Spec.DeepCopyInto(&prometheus.Spec) + return nil + }) + + if err != nil { + r.Log.Error(err, "Failed to create/update prometheus instance") + return err + } + r.Log.Info("Prometheus instance creation succeeded", "Name", prometheus.Name) + + return nil +} + +func (r *OCSInitializationReconciler) reconcileAlertManager(initialData *ocsv1.OCSInitialization) error { + var err error + + alertManager := &promv1.Alertmanager{} + alertManager.Name = "odf-alertmanager" + alertManager.Namespace = initialData.Namespace + + _, err = ctrl.CreateOrUpdate(r.ctx, r.Client, alertManager, func() error { + if err := ctrl.SetControllerReference(initialData, alertManager, r.Scheme); err != nil { + return err + } + util.AddAnnotation(alertManager, "prometheus", "odf-prometheus") + templates.AlertmanagerTemplate.Spec.DeepCopyInto(&alertManager.Spec) + return nil + }) + if err != nil { + r.Log.Error(err, "Failed to create/update alertManager instance") + return err + } + r.Log.Info("AlertManager instance creation succeeded", "Name", alertManager.Name) + return nil +} + +func (r *OCSInitializationReconciler) reconcilePrometheusProxyNetworkPolicy(initialData *ocsv1.OCSInitialization) error { + var err error + + promethuesProxyNetworkPolicy := &networkingv1.NetworkPolicy{} + promethuesProxyNetworkPolicy.Name = "prometheus-proxy-rule" + promethuesProxyNetworkPolicy.Namespace = initialData.Namespace + + _, err = ctrl.CreateOrUpdate(r.ctx, r.Client, promethuesProxyNetworkPolicy, func() error { + if err := ctrl.SetControllerReference(initialData, promethuesProxyNetworkPolicy, r.Scheme); err != nil { + return err + } + templates.PrometheusProxyNetworkPolicyTemplate.Spec.DeepCopyInto(&promethuesProxyNetworkPolicy.Spec) + return nil + }) + if err != nil { + r.Log.Error(err, "Failed to create/update Prometheus proxy network policy") + return err + } + r.Log.Info("Prometheus proxy network policy creation succeeded", "Name", promethuesProxyNetworkPolicy.Name) + return nil +} + +func (r *OCSInitializationReconciler) reconcileK8sMetricsServiceMonitor(initialData *ocsv1.OCSInitialization) error { + var err error + + k8sMetricsServiceMonitor := &promv1.ServiceMonitor{} + k8sMetricsServiceMonitor.Name = "k8s-metrics-service-monitor" + k8sMetricsServiceMonitor.Namespace = initialData.Namespace + + _, err = ctrl.CreateOrUpdate(r.ctx, r.Client, k8sMetricsServiceMonitor, func() error { + if err := ctrl.SetControllerReference(initialData, k8sMetricsServiceMonitor, r.Scheme); err != nil { + return err + } + templates.K8sMetricsServiceMonitorTemplate.Spec.DeepCopyInto(&k8sMetricsServiceMonitor.Spec) + return nil + }) + if err != nil { + r.Log.Error(err, "Failed to create/update K8s Metrics Service Monitor") + return err + } + r.Log.Info("K8s Metrics Service Monitor creation succeeded", "Name", k8sMetricsServiceMonitor.Name) + return nil + +} diff --git a/controllers/platform/platform_detection.go b/controllers/platform/platform_detection.go index 79dc1ee823..915c7d2afd 100644 --- a/controllers/platform/platform_detection.go +++ b/controllers/platform/platform_detection.go @@ -33,8 +33,9 @@ var ( // platform is used to get the PlatformType of the running cluster in a thread-safe manner // It is a singleton which is initialized exactly once via Detect() function call. type platform struct { - isOpenShift bool - platform configv1.PlatformType + isOpenShift bool + platform configv1.PlatformType + infrastructure *configv1.Infrastructure } // SetFakePlatformInstanceForTesting can be used to fake a Platform while testing. @@ -88,11 +89,12 @@ func Detect() { } } } - if platformInstance.isOpenShift { - if infrastructure, err := configv1client(cfg).Infrastructures().Get(context.TODO(), "cluster", metav1.GetOptions{}); err != nil { + if infrastructure, err := configv1client(cfg).Infrastructures().Get(context.TODO(), "cluster", metav1.GetOptions{}); err == nil { platformInstance.platform = infrastructure.Status.PlatformStatus.Type + platformInstance.infrastructure = infrastructure } + } }) } @@ -160,3 +162,16 @@ func SkipObjectStore(p configv1.PlatformType) bool { } return false } + +func IsPlatformROSAHCP() (bool, error) { + if platformInstance.platform == configv1.AWSPlatformType { + if platformInstance.infrastructure.Status.ControlPlaneTopology == configv1.ExternalTopologyMode { + for _, resourceTags := range platformInstance.infrastructure.Status.PlatformStatus.AWS.ResourceTags { + if resourceTags.Key == "red-hat-clustertype" && resourceTags.Value == "rosa" { + return true, nil + } + } + } + } + return false, nil +} diff --git a/controllers/util/util.go b/controllers/util/util.go index 278ecb8a7d..d98e9befdd 100644 --- a/controllers/util/util.go +++ b/controllers/util/util.go @@ -2,6 +2,7 @@ package util import ( ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func RemoveDuplicatesFromStringSlice(slice []string) []string { @@ -54,3 +55,29 @@ func Find[T any](list []T, f func(item *T) bool) *T { } return nil } + +func AddAnnotation(obj metav1.Object, key string, value string) bool { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + obj.SetAnnotations(annotations) + } + if oldValue, exist := annotations[key]; !exist || oldValue != value { + annotations[key] = value + return true + } + return false +} + +func AddLabels(obj metav1.Object, key string, value string) bool { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + obj.SetLabels(labels) + } + if oldValue, exist := labels[key]; !exist || oldValue != value { + labels[key] = value + return true + } + return false +} diff --git a/deploy/csv-templates/ocs-operator.csv.yaml.in b/deploy/csv-templates/ocs-operator.csv.yaml.in index f91fe072d3..63b28be050 100644 --- a/deploy/csv-templates/ocs-operator.csv.yaml.in +++ b/deploy/csv-templates/ocs-operator.csv.yaml.in @@ -328,6 +328,19 @@ spec: - get - list - watch + - apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - monitoring.coreos.com resources: @@ -340,6 +353,28 @@ spec: - list - update - watch + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - get + - list + - update + - watch - apiGroups: - noobaa.io resources: diff --git a/deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role-binding.yaml b/deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role-binding.yaml new file mode 100644 index 0000000000..195b0a36a3 --- /dev/null +++ b/deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8s-metrics-sm-prometheus-k8s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8s-metrics-sm-prometheus-k8s +subjects: + - kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring diff --git a/deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role.yaml b/deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role.yaml new file mode 100644 index 0000000000..050f85a3e1 --- /dev/null +++ b/deploy/ocs-operator/manifests/k8s-metrics-servicemonitor-role.yaml @@ -0,0 +1,51 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8s-metrics-sm-prometheus-k8s +rules: + - verbs: + - get + apiGroups: + - '' + resources: + - nodes/metrics + - verbs: + - get + nonResourceURLs: + - /metrics + - verbs: + - create + apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + - verbs: + - create + apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + - verbs: + - get + apiGroups: + - '' + resources: + - namespaces + - verbs: + - use + apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot + - verbs: + - list + - watch + - get + apiGroups: + - '' + resources: + - pods + - endpoints + - services diff --git a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml index 9376976fc5..541354ef8f 100644 --- a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml +++ b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml @@ -2093,6 +2093,19 @@ spec: - get - list - watch + - apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - monitoring.coreos.com resources: @@ -2105,6 +2118,28 @@ spec: - list - update - watch + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - get + - list + - update + - watch - apiGroups: - noobaa.io resources: diff --git a/deploy/ocs-operator/manifests/odf-prometheus-role-binding.yaml b/deploy/ocs-operator/manifests/odf-prometheus-role-binding.yaml new file mode 100644 index 0000000000..5858728e9a --- /dev/null +++ b/deploy/ocs-operator/manifests/odf-prometheus-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: odf-prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: odf-prometheus +subjects: +- kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring diff --git a/deploy/ocs-operator/manifests/odf-prometheus-role.yaml b/deploy/ocs-operator/manifests/odf-prometheus-role.yaml new file mode 100644 index 0000000000..58f679e113 --- /dev/null +++ b/deploy/ocs-operator/manifests/odf-prometheus-role.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: odf-prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/main.go b/main.go index 711a6a5381..1d97545cde 100644 --- a/main.go +++ b/main.go @@ -47,6 +47,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" storagev1 "k8s.io/api/storage/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -92,6 +93,7 @@ func init() { utilruntime.Must(clusterv1alpha1.AddToScheme(scheme)) utilruntime.Must(operatorsv1alpha1.AddToScheme(scheme)) utilruntime.Must(nadscheme.AddToScheme(scheme)) + utilruntime.Must(networkingv1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/rbac/k8s-metrics-servicemonitor-role-binding.yaml b/rbac/k8s-metrics-servicemonitor-role-binding.yaml new file mode 100644 index 0000000000..195b0a36a3 --- /dev/null +++ b/rbac/k8s-metrics-servicemonitor-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8s-metrics-sm-prometheus-k8s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8s-metrics-sm-prometheus-k8s +subjects: + - kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring diff --git a/rbac/k8s-metrics-servicemonitor-role.yaml b/rbac/k8s-metrics-servicemonitor-role.yaml new file mode 100644 index 0000000000..050f85a3e1 --- /dev/null +++ b/rbac/k8s-metrics-servicemonitor-role.yaml @@ -0,0 +1,51 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8s-metrics-sm-prometheus-k8s +rules: + - verbs: + - get + apiGroups: + - '' + resources: + - nodes/metrics + - verbs: + - get + nonResourceURLs: + - /metrics + - verbs: + - create + apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + - verbs: + - create + apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + - verbs: + - get + apiGroups: + - '' + resources: + - namespaces + - verbs: + - use + apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot + - verbs: + - list + - watch + - get + apiGroups: + - '' + resources: + - pods + - endpoints + - services diff --git a/rbac/odf-prometheus-role-binding.yaml b/rbac/odf-prometheus-role-binding.yaml new file mode 100644 index 0000000000..5858728e9a --- /dev/null +++ b/rbac/odf-prometheus-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: odf-prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: odf-prometheus +subjects: +- kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring diff --git a/rbac/odf-prometheus-role.yaml b/rbac/odf-prometheus-role.yaml new file mode 100644 index 0000000000..58f679e113 --- /dev/null +++ b/rbac/odf-prometheus-role.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: odf-prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/templates/alertmanager.go b/templates/alertmanager.go new file mode 100644 index 0000000000..15fab8fe3d --- /dev/null +++ b/templates/alertmanager.go @@ -0,0 +1,14 @@ +package templates + +import ( + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" + "k8s.io/utils/ptr" +) + +var AlertmanagerTemplate = promv1.Alertmanager{ + Spec: promv1.AlertmanagerSpec{ + Replicas: ptr.To(int32(1)), + Resources: defaults.MonitoringResources["alertmanager"], + }, +} diff --git a/templates/k8smetricsservicemonitor.go b/templates/k8smetricsservicemonitor.go new file mode 100644 index 0000000000..93587e8499 --- /dev/null +++ b/templates/k8smetricsservicemonitor.go @@ -0,0 +1,70 @@ +package templates + +import ( + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var params = map[string][]string{ + "match[]": { + "{__name__='kube_node_status_condition'}", + "{__name__='kube_persistentvolume_info'}", + "{__name__='kube_storageclass_info'}", + "{__name__='kube_persistentvolumeclaim_info'}", + "{__name__='kube_deployment_spec_replicas'}", + "{__name__='kube_pod_status_phase'}", + "{__name__='kubelet_volume_stats_capacity_bytes'}", + "{__name__='kubelet_volume_stats_used_bytes'}", + "{__name__='node_disk_read_time_seconds_total'}", + "{__name__='node_disk_write_time_seconds_total'}", + "{__name__='node_disk_reads_completed_total'}", + "{__name__='node_disk_writes_completed_total'}", + }, +} + +var K8sMetricsServiceMonitorTemplate = promv1.ServiceMonitor{ + Spec: promv1.ServiceMonitorSpec{ + Endpoints: []promv1.Endpoint{ + { + Port: "web", + Path: "/federate", + Scheme: "https", + ScrapeTimeout: "1m", + Interval: "2m", + HonorLabels: true, + MetricRelabelConfigs: []*promv1.RelabelConfig{ + { + Action: "labeldrop", + Regex: "prometheus_replica", + }, + }, + RelabelConfigs: []*promv1.RelabelConfig{ + { + Action: "replace", + Regex: "prometheus-k8s-.*", + Replacement: "", + SourceLabels: []promv1.LabelName{ + "pod", + }, + TargetLabel: "pod", + }, + }, + TLSConfig: &promv1.TLSConfig{ + SafeTLSConfig: promv1.SafeTLSConfig{ + InsecureSkipVerify: true, + }, + }, + Params: params, + BearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, + }, + NamespaceSelector: promv1.NamespaceSelector{ + MatchNames: []string{"openshift-monitoring"}, + }, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/component": "prometheus", + }, + }, + }, +} diff --git a/templates/prometheus.go b/templates/prometheus.go new file mode 100644 index 0000000000..552023886d --- /dev/null +++ b/templates/prometheus.go @@ -0,0 +1,92 @@ +package templates + +import ( + "fmt" + + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +var resourceSelector = metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "odf-prometheus", + }, +} + +var ( + KubeRBACProxyPortNumber = 9339 + PrometheusKubeRBACProxyConfigMapName = "prometheus-kube-rbac-proxy-config" +) + +var PrometheusTemplate = promv1.Prometheus{ + Spec: promv1.PrometheusSpec{ + CommonPrometheusFields: promv1.CommonPrometheusFields{ + ServiceAccountName: "prometheus-k8s", + ServiceMonitorSelector: &resourceSelector, + PodMonitorSelector: &resourceSelector, + ListenLocal: true, + Resources: defaults.MonitoringResources["prometheus"], + Containers: []corev1.Container{{ + Name: "kube-rbac-proxy", + Image: "gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0", + Args: []string{ + fmt.Sprintf("--secure-listen-address=0.0.0.0:%d", KubeRBACProxyPortNumber), + "--upstream=http://127.0.0.1:9090/", + "--logtostderr=true", + "--v=10", + "--tls-cert-file=/etc/tls-secret/tls.crt", + "--tls-private-key-file=/etc/tls-secret/tls.key", + "--client-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt", + "--config-file=/etc/kube-rbac-config/config-file.json", + }, + Ports: []corev1.ContainerPort{{ + Name: "https", + ContainerPort: int32(KubeRBACProxyPortNumber), + }}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "serving-cert", + MountPath: "/etc/tls-secret", + }, + { + Name: "kube-rbac-config", + MountPath: "/etc/kube-rbac-config", + }, + }, + Resources: defaults.MonitoringResources["kube-rbac-proxy"], + }}, + Volumes: []corev1.Volume{ + { + Name: "serving-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "prometheus-serving-cert-secret", + }, + }, + }, + { + Name: "kube-rbac-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: PrometheusKubeRBACProxyConfigMapName, + }, + }, + }, + }, + }, + }, + RuleSelector: &resourceSelector, + EnableAdminAPI: false, + Alerting: &promv1.AlertingSpec{ + Alertmanagers: []promv1.AlertmanagerEndpoints{{ + Namespace: "", + Name: "alertmanager-operated", + Port: intstr.FromString("web"), + }}, + }, + }, +} diff --git a/templates/prometheuskuberbacproxyconfig.go b/templates/prometheuskuberbacproxyconfig.go new file mode 100644 index 0000000000..c1c4775981 --- /dev/null +++ b/templates/prometheuskuberbacproxyconfig.go @@ -0,0 +1,36 @@ +package templates + +import ( + "encoding/json" + + corev1 "k8s.io/api/core/v1" +) + +var KubeRBACProxyConfigMap = corev1.ConfigMap{ + Data: map[string]string{ + "config-file.json": (func() string { + config := struct { + Authorization struct { + Static [2]struct { + Path string `json:"path"` + ResourceRequest bool `json:"resourceRequest"` + Verb string `json:"verb"` + } `json:"static"` + } `json:"authorization"` + }{} + + item := &config.Authorization.Static[0] + item.Verb = "get" + item.Path = "/metrics" + item.ResourceRequest = false + + item = &config.Authorization.Static[1] + item.Verb = "get" + item.Path = "/federate" + item.ResourceRequest = false + + raw, _ := json.Marshal(config) + return string(raw) + })(), + }, +} diff --git a/templates/prometheusproxynetworkpolicy.go b/templates/prometheusproxynetworkpolicy.go new file mode 100644 index 0000000000..6c6269e9fc --- /dev/null +++ b/templates/prometheusproxynetworkpolicy.go @@ -0,0 +1,42 @@ +package templates + +import ( + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +var ( + prometheusProxyPort = intstr.FromInt(KubeRBACProxyPortNumber) + prometheusProxyProtocol = corev1.ProtocolTCP +) + +var PrometheusProxyNetworkPolicyTemplate = netv1.NetworkPolicy{ + Spec: netv1.NetworkPolicySpec{ + Ingress: []netv1.NetworkPolicyIngressRule{ + { + Ports: []netv1.NetworkPolicyPort{ + { + Port: &prometheusProxyPort, + Protocol: &prometheusProxyProtocol, + }, + }, + }, + }, + PolicyTypes: []netv1.PolicyType{ + netv1.PolicyTypeIngress, + }, + PodSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "prometheus", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "odf-prometheus", + }, + }, + }, + }, + }, +}