From 51f5c0f69e664dcb8169990a719c2248e1333514 Mon Sep 17 00:00:00 2001 From: Umanga Chapagain Date: Fri, 8 Dec 2023 17:48:29 +0530 Subject: [PATCH 1/4] remove ocs-metrics-exporter from CSV ocs-metrics-exporter is now deployed with every StorageCluster. We do not need to deploy it during operator installation anymore. Signed-off-by: Umanga Chapagain --- hack/install-ocs.sh | 1 - tools/csv-merger/csv-merger.go | 88 ++-------------------------------- 2 files changed, 4 insertions(+), 85 deletions(-) diff --git a/hack/install-ocs.sh b/hack/install-ocs.sh index de8b7daad1..335e38e8f3 100755 --- a/hack/install-ocs.sh +++ b/hack/install-ocs.sh @@ -17,5 +17,4 @@ fi "$OPERATOR_SDK" run bundle "$BUNDLE_FULL_IMAGE_NAME" --timeout=10m --security-context-config restricted -n "$INSTALL_NAMESPACE" oc wait --timeout=5m --for condition=Available -n "$INSTALL_NAMESPACE" deployment ocs-operator -oc wait --timeout=5m --for condition=Available -n "$INSTALL_NAMESPACE" deployment ocs-metrics-exporter oc wait --timeout=5m --for condition=Available -n "$INSTALL_NAMESPACE" deployment rook-ceph-operator diff --git a/tools/csv-merger/csv-merger.go b/tools/csv-merger/csv-merger.go index 00736539b5..2fdae66c54 100644 --- a/tools/csv-merger/csv-merger.go +++ b/tools/csv-merger/csv-merger.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -134,6 +133,10 @@ func unmarshalCSV(filePath string) *csvv1.ClusterServiceVersion { // inject custom ENV VARS. if strings.Contains(csv.Name, "ocs") || strings.Contains(csv.Name, "ics") { vars := []corev1.EnvVar{ + { + Name: "OCS_METRICS_EXPORTER_IMAGE", + Value: *ocsMetricsExporterImage, + }, { Name: "ROOK_CEPH_IMAGE", Value: *rookContainerImage, @@ -540,13 +543,6 @@ func generateUnifiedCSV() *csvv1.ClusterServiceVersion { } - // Add metrics exporter deployment to CSV - metricExporterStrategySpec := csvv1.StrategyDeploymentSpec{ - Name: "ocs-metrics-exporter", - Spec: getMetricsExporterDeployment(), - } - templateStrategySpec.DeploymentSpecs = append(templateStrategySpec.DeploymentSpecs, metricExporterStrategySpec) - // Add tolerations to deployments for i := range templateStrategySpec.DeploymentSpecs { d := &templateStrategySpec.DeploymentSpecs[i] @@ -921,82 +917,6 @@ func copyManifests() { } } -func getMetricsExporterDeployment() appsv1.DeploymentSpec { - replica := int32(1) - privileged := false - noRoot := true - deployment := appsv1.DeploymentSpec{ - Replicas: &replica, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/component": "ocs-metrics-exporter", - "app.kubernetes.io/name": "ocs-metrics-exporter", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/component": "ocs-metrics-exporter", - "app.kubernetes.io/name": "ocs-metrics-exporter", - "app.kubernetes.io/version": "0.0.1", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "ocs-metrics-exporter", - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - RunAsNonRoot: &noRoot, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "ceph-config", - MountPath: "/etc/ceph", - }, - }, - Image: *ocsMetricsExporterImage, - Command: []string{"/usr/local/bin/metrics-exporter"}, - Args: []string{"--namespaces=$(WATCH_NAMESPACE)"}, - Env: []corev1.EnvVar{ - { - Name: "WATCH_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - }, - Ports: []corev1.ContainerPort{ - { - ContainerPort: 8080, - }, - { - ContainerPort: 8081, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "ceph-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "ocs-metrics-exporter-ceph-conf", - }, - }, - }, - }, - }, - ServiceAccountName: "ocs-metrics-exporter", - }, - }, - } - return deployment -} - func main() { flag.Parse() From 47ce07978c02c5b8ad72e2be9524b4159afb3618 Mon Sep 17 00:00:00 2001 From: Umanga Chapagain Date: Fri, 8 Dec 2023 17:50:28 +0530 Subject: [PATCH 2/4] generated CSV without ocs-metrics-exporter deploy Signed-off-by: Umanga Chapagain --- .../ocs-operator.clusterserviceversion.yaml | 49 +------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml index a997793373..cedff714f0 100644 --- a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml +++ b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml @@ -3076,6 +3076,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] + - name: OCS_METRICS_EXPORTER_IMAGE + value: quay.io/ocs-dev/ocs-metrics-exporter:latest - name: ROOK_CEPH_IMAGE value: docker.io/rook/ceph:v1.12.0.545.geacc7e744 - name: CEPH_IMAGE @@ -3267,53 +3269,6 @@ spec: name: default-config-dir - emptyDir: {} name: webhook-cert - - name: ocs-metrics-exporter - spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/component: ocs-metrics-exporter - app.kubernetes.io/name: ocs-metrics-exporter - strategy: {} - template: - metadata: - labels: - app.kubernetes.io/component: ocs-metrics-exporter - app.kubernetes.io/name: ocs-metrics-exporter - app.kubernetes.io/version: 0.0.1 - spec: - containers: - - args: - - --namespaces=$(WATCH_NAMESPACE) - command: - - /usr/local/bin/metrics-exporter - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: quay.io/ocs-dev/ocs-metrics-exporter:latest - name: ocs-metrics-exporter - ports: - - containerPort: 8080 - - containerPort: 8081 - resources: {} - securityContext: - privileged: false - runAsNonRoot: true - volumeMounts: - - mountPath: /etc/ceph - name: ceph-config - serviceAccountName: ocs-metrics-exporter - tolerations: - - effect: NoSchedule - key: node.ocs.openshift.io/storage - operator: Equal - value: "true" - volumes: - - configMap: - name: ocs-metrics-exporter-ceph-conf - name: ceph-config permissions: - rules: - apiGroups: From aa3809159599f23d76be0ed3d1e6b6a16830a41e Mon Sep 17 00:00:00 2001 From: Umanga Chapagain Date: Mon, 11 Dec 2023 18:08:29 +0530 Subject: [PATCH 3/4] remove ocs-metrics-exporter serviceaccount & RBACs ocs-metrics-exporter is no longer deployed via the CSV at operator install time. So, we do not need to generate the "ocs-metrics-exporter" serviceaccount and related ClusterRole and ClusterRoleBindings. Signed-off-by: Umanga Chapagain --- config/rbac/exporter-role.yaml | 68 -------------------------- config/rbac/exporter-role_binding.yaml | 12 ----- config/rbac/kustomization.yaml | 2 - hack/source-manifests.sh | 2 +- tools/csv-merger/csv-merger.go | 11 ----- 5 files changed, 1 insertion(+), 94 deletions(-) delete mode 100644 config/rbac/exporter-role.yaml delete mode 100644 config/rbac/exporter-role_binding.yaml diff --git a/config/rbac/exporter-role.yaml b/config/rbac/exporter-role.yaml deleted file mode 100644 index 8cbdc41b91..0000000000 --- a/config/rbac/exporter-role.yaml +++ /dev/null @@ -1,68 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ocs-metrics-exporter -rules: -- apiGroups: - - ceph.rook.io - resources: - - cephobjectstores - - cephblockpools - - cephclusters - - cephrbdmirrors - verbs: - - get - - list - - watch -- apiGroups: - - quota.openshift.io - resources: - - clusterresourcequotas - verbs: - - get - - list - - watch -- apiGroups: - - objectbucket.io - resources: - - objectbuckets - - objectbucketclaims - verbs: - - get - - list -- apiGroups: - - "" - resources: - - configmaps - - secrets - verbs: - - get - - list -- apiGroups: - - "" - resources: - - persistentvolumes - - persistentvolumeclaims - - pods - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - ocs.openshift.io - resources: - - storageconsumers - - storageclusters - verbs: - - get - - list - - watch diff --git a/config/rbac/exporter-role_binding.yaml b/config/rbac/exporter-role_binding.yaml deleted file mode 100644 index 34e4640ab9..0000000000 --- a/config/rbac/exporter-role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ocs-metrics-exporter -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ocs-metrics-exporter -subjects: -- kind: ServiceAccount - name: ocs-metrics-exporter - namespace: openshift-storage diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 96d44a2cbd..45decf8844 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -3,8 +3,6 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -- exporter-role.yaml -- exporter-role_binding.yaml # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. diff --git a/hack/source-manifests.sh b/hack/source-manifests.sh index 73fa2299bd..1236cde8e9 100755 --- a/hack/source-manifests.sh +++ b/hack/source-manifests.sh @@ -69,7 +69,7 @@ function gen_ocs_csv() { pushd config/manager $KUSTOMIZE edit set image ocs-dev/ocs-operator="$OCS_IMAGE" popd - $KUSTOMIZE build config/manifests/ocs-operator | $OPERATOR_SDK generate bundle -q --overwrite=false --output-dir deploy/ocs-operator --kustomize-dir config/manifests/ocs-operator --package ocs-operator --version "$CSV_VERSION" --extra-service-accounts=ocs-metrics-exporter + $KUSTOMIZE build config/manifests/ocs-operator | $OPERATOR_SDK generate bundle -q --overwrite=false --output-dir deploy/ocs-operator --kustomize-dir config/manifests/ocs-operator --package ocs-operator --version "$CSV_VERSION" mv deploy/ocs-operator/manifests/*clusterserviceversion.yaml $OCS_CSV cp config/crd/bases/* $ocs_crds_outdir } diff --git a/tools/csv-merger/csv-merger.go b/tools/csv-merger/csv-merger.go index 2fdae66c54..cc418fff1e 100644 --- a/tools/csv-merger/csv-merger.go +++ b/tools/csv-merger/csv-merger.go @@ -21,7 +21,6 @@ import ( ocsversion "github.com/red-hat-storage/ocs-operator/v4/version" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -549,16 +548,6 @@ func generateUnifiedCSV() *csvv1.ClusterServiceVersion { d.Spec.Template.Spec.Tolerations = ocsNodeToleration } - templateStrategySpec.ClusterPermissions = append(templateStrategySpec.ClusterPermissions, csvv1.StrategyDeploymentPermissions{ - ServiceAccountName: "ocs-metrics-exporter", - Rules: []rbac.PolicyRule{ - { - APIGroups: []string{"monitoring.coreos.com"}, - Resources: []string{"*"}, - Verbs: []string{"*"}, - }, - }, - }) fmt.Println(templateStrategySpec.DeploymentSpecs) // Set correct csv versions and name From bd260f58f691f2ec179ee4fd5c6273fa3f91a79b Mon Sep 17 00:00:00 2001 From: Umanga Chapagain Date: Mon, 11 Dec 2023 18:12:10 +0530 Subject: [PATCH 4/4] update bundle with generated changes These changes include removal of ocs-metrics-exporter ServiceAccount and it's related RBACs from the operator bundle. Signed-off-by: Umanga Chapagain --- deploy/csv-templates/ocs-operator.csv.yaml.in | 65 ----------------- .../ocs-operator.clusterserviceversion.yaml | 73 ------------------- 2 files changed, 138 deletions(-) diff --git a/deploy/csv-templates/ocs-operator.csv.yaml.in b/deploy/csv-templates/ocs-operator.csv.yaml.in index 62ec92a56b..e7d8c3beba 100644 --- a/deploy/csv-templates/ocs-operator.csv.yaml.in +++ b/deploy/csv-templates/ocs-operator.csv.yaml.in @@ -156,71 +156,6 @@ spec: install: spec: clusterPermissions: - - rules: - - apiGroups: - - ceph.rook.io - resources: - - cephobjectstores - - cephblockpools - - cephclusters - - cephrbdmirrors - verbs: - - get - - list - - watch - - apiGroups: - - quota.openshift.io - resources: - - clusterresourcequotas - verbs: - - get - - list - - watch - - apiGroups: - - objectbucket.io - resources: - - objectbuckets - - objectbucketclaims - verbs: - - get - - list - - apiGroups: - - "" - resources: - - configmaps - - secrets - verbs: - - get - - list - - apiGroups: - - "" - resources: - - persistentvolumes - - persistentvolumeclaims - - pods - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch - - apiGroups: - - ocs.openshift.io - resources: - - storageconsumers - - storageclusters - verbs: - - get - - list - - watch - serviceAccountName: ocs-metrics-exporter - rules: - apiGroups: - apiextensions.k8s.io diff --git a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml index cedff714f0..26854ca6bd 100644 --- a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml +++ b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml @@ -1876,71 +1876,6 @@ spec: install: spec: clusterPermissions: - - rules: - - apiGroups: - - ceph.rook.io - resources: - - cephobjectstores - - cephblockpools - - cephclusters - - cephrbdmirrors - verbs: - - get - - list - - watch - - apiGroups: - - quota.openshift.io - resources: - - clusterresourcequotas - verbs: - - get - - list - - watch - - apiGroups: - - objectbucket.io - resources: - - objectbuckets - - objectbucketclaims - verbs: - - get - - list - - apiGroups: - - "" - resources: - - configmaps - - secrets - verbs: - - get - - list - - apiGroups: - - "" - resources: - - persistentvolumes - - persistentvolumeclaims - - pods - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch - - apiGroups: - - ocs.openshift.io - resources: - - storageconsumers - - storageclusters - verbs: - - get - - list - - watch - serviceAccountName: ocs-metrics-exporter - rules: - apiGroups: - apiextensions.k8s.io @@ -3043,14 +2978,6 @@ spec: verbs: - use serviceAccountName: rook-ceph-system - - rules: - - apiGroups: - - monitoring.coreos.com - resources: - - '*' - verbs: - - '*' - serviceAccountName: ocs-metrics-exporter deployments: - name: ocs-operator spec: