Add sharding-exporter

README.md

@@ -64,7 +64,8 @@ Read chapter 4 of the full [thesis](https://github.com/timebertt/thesis-controll
 - [samples-generator](webhosting-operator/cmd/samples-generator): a tool for generating a given amount of random `Website` objects
 - [monitoring setup](hack/config/monitoring): a setup for monitoring and measuring load test experiments for the sample operator
   - includes [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus)
-  - [webhosting-exporter](webhosting-operator/config/monitoring/webhosting-exporter) (based on the [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) [custom resource metrics feature](https://github.com/kubernetes/kube-state-metrics/blob/main/docs/customresourcestate-metrics.md)) for metrics on the state of the webhosting-operator's API objects
+  - [sharding-exporter](config/monitoring/sharding-exporter): (based on the [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) [custom resource metrics feature](https://github.com/kubernetes/kube-state-metrics/blob/main/docs/customresourcestate-metrics.md)) for metrics on the state of shards
+  - [webhosting-exporter](webhosting-operator/config/monitoring/webhosting-exporter) for metrics on the state of the webhosting-operator's API objects (similar to above)
   - [grafana](https://github.com/grafana/grafana) along with some dashboards for [controller-runtime](hack/config/monitoring/default/dashboards) and [webhosting-operator and sharding](webhosting-operator/config/monitoring/default/dashboards)
 - [experiment](webhosting-operator/cmd/experiment): a tool (based on controller-runtime) for executing load test scenarios for the webhosting-operator
 - [measure](webhosting-operator/cmd/measure): a tool for retrieving configurable measurements from prometheus and storing them in csv-formatted files for further analysis (with `numpy`) and visualization (with `matplotlib`)

config/monitoring/default/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../sharder
+- ../sharding-exporter

config/monitoring/sharding-exporter/clusterrole.yaml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: sharding:exporter
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - sharding.timebertt.dev
+  resources:
+  - clusterrings
+  verbs:
+  - get
+  - list
+  - watch

config/monitoring/sharding-exporter/clusterrolebinding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: sharding:exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sharding:exporter
+subjects:
+- kind: ServiceAccount
+  name: sharding-exporter

config/monitoring/sharding-exporter/config.yaml

@@ -0,0 +1,71 @@
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+  # shard metrics
+  - metricNamePrefix: kube_shard
+    groupVersionKind:
+      group: coordination.k8s.io
+      version: v1
+      kind: Lease
+    labelsFromPath:
+      namespace: [metadata, namespace]
+      shard: [metadata, name]
+      clusterring: [metadata, labels, sharding.alpha.kubernetes.io/clusterring]
+    metrics:
+    - name: info
+      help: "Information about a Shard"
+      each:
+        type: Info
+        info:
+          labelsFromPath:
+            uid: [metadata, uid]
+    - name: state
+      help: "The Shard's current state"
+      each:
+        type: StateSet
+        stateSet:
+          labelName: state
+          path: [metadata, labels, sharding.alpha.kubernetes.io/state]
+          list: [orphaned, dead, uncertain, expired, ready, unknown]
+      # The usual leader election leases don't have the state label making the generator log errors.
+      # Hence, decrease verbosity of such errors to reduce distraction.
+      errorLogV: 4
+  # clusterring metrics
+  - metricNamePrefix: kube_clusterring
+    groupVersionKind:
+      group: sharding.timebertt.dev
+      version: v1alpha1
+      kind: ClusterRing
+    labelsFromPath:
+      clusterring: [metadata, name]
+      uid: [metadata, uid]
+    metrics:
+    - name: info
+      help: "Information about a ClusterRing"
+      each:
+        type: Info
+        info: {}
+    - name: metadata_generation
+      help: "The generation of a ClusterRing"
+      each:
+        type: Gauge
+        gauge:
+          path: [metadata, generation]
+    - name: observed_generation
+      help: "The latest generation observed by the ClusterRing controller"
+      each:
+        type: Gauge
+        gauge:
+          path: [status, observedGeneration]
+    - name: status_shards
+      help: "The ClusterRing's total number of shards"
+      each:
+        type: Gauge
+        gauge:
+          path: [status, shards]
+    - name: status_shards
+      help: "The ClusterRing's total number of available shards"
+      each:
+        type: Gauge
+        gauge:
+          path: [status, availableShards]

config/monitoring/sharding-exporter/deployment.yaml

@@ -0,0 +1,98 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sharding-exporter
+spec:
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: sharding-exporter
+    spec:
+      automountServiceAccountToken: true
+      containers:
+      - args:
+        - --host=127.0.0.1
+        - --port=8081
+        - --telemetry-host=127.0.0.1
+        - --telemetry-port=8082
+        - --custom-resource-state-only
+        - --custom-resource-state-config-file=/etc/kube-state-metrics/config/config.yaml
+        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.9.2
+        name: sharding-exporter
+        resources:
+          limits:
+            cpu: 250m
+            memory: 400Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsUser: 65534
+        volumeMounts:
+        - name: config
+          mountPath: /etc/kube-state-metrics/config
+      - args:
+        - --logtostderr
+        - --secure-listen-address=:8443
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+        - --upstream=http://127.0.0.1:8081/
+        image: quay.io/brancz/kube-rbac-proxy:v0.14.0
+        name: kube-rbac-proxy-main
+        ports:
+        - containerPort: 8443
+          name: https-main
+        resources:
+          limits:
+            cpu: 40m
+            memory: 40Mi
+          requests:
+            cpu: 20m
+            memory: 20Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsGroup: 65532
+          runAsNonRoot: true
+          runAsUser: 65532
+      - args:
+        - --logtostderr
+        - --secure-listen-address=:9443
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+        - --upstream=http://127.0.0.1:8082/
+        image: quay.io/brancz/kube-rbac-proxy:v0.14.0
+        name: kube-rbac-proxy-self
+        ports:
+        - containerPort: 9443
+          name: https-self
+        resources:
+          limits:
+            cpu: 20m
+            memory: 40Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsGroup: 65532
+          runAsNonRoot: true
+          runAsUser: 65532
+      volumes:
+      - name: config
+        configMap:
+          name: sharding-exporter-config
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: sharding-exporter

config/monitoring/sharding-exporter/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: sharding-system
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/name: controller-sharding
+    app.kubernetes.io/component: sharding-exporter
+
+resources:
+- clusterrole.yaml
+- clusterrolebinding.yaml
+- rbac-proxy_clusterrole.yaml
+- rbac-proxy_clusterrolebinding.yaml
+- serviceaccount.yaml
+- service.yaml
+- deployment.yaml
+- servicemonitor.yaml
+
+configMapGenerator:
+- name: sharding-exporter-config
+  files:
+  - config.yaml

config/monitoring/sharding-exporter/rbac-proxy_clusterrole.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: sharding:exporter:rbac-proxy
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create

config/monitoring/sharding-exporter/rbac-proxy_clusterrolebinding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: sharding:exporter:rbac-proxy
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sharding:exporter:rbac-proxy
+subjects:
+- kind: ServiceAccount
+  name: sharding-exporter

config/monitoring/sharding-exporter/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sharding-exporter
+spec:
+  clusterIP: None
+  ports:
+  - name: https-main
+    port: 8443
+    targetPort: https-main
+  - name: https-self
+    port: 9443
+    targetPort: https-self

config/monitoring/sharding-exporter/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sharding-exporter
+automountServiceAccountToken: false

config/monitoring/sharding-exporter/servicemonitor.yaml

@@ -0,0 +1,28 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: sharding-exporter
+spec:
+  jobLabel: app.kubernetes.io/component
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 10s
+    port: https-main
+    relabelings:
+    - action: labeldrop
+      regex: (pod|service|endpoint|namespace)
+    scheme: https
+    scrapeTimeout: 10s
+    tlsConfig:
+      insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    port: https-self
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: controller-sharding
+      app.kubernetes.io/component: sharding-exporter

hack/config/skaffold.yaml

@@ -212,7 +212,7 @@ manifests:
   kustomize:
     paths:
       - config/default
-      - config/monitoring
+      - config/monitoring/default
 deploy:
   kubectl:
     flags: