From 6977316a87c62d6e647ef83c520452af9d0a5ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Fri, 13 Sep 2024 11:35:08 +0200 Subject: [PATCH] jsonnet: Ingester automated downscale v2 (#9271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Automated downscale of ingesters now uses read-only ingesters and mirroring feature of rollout-operator. Signed-off-by: Peter Štibraný * Move new autoscaling to new file. Signed-off-by: Peter Štibraný * Add ingester-automated-downscale_v2.libsonnet. Signed-off-by: Peter Štibraný * Add missing test file. Signed-off-by: Peter Štibraný * Don't enable ingest-storage ingester autoscaling annotations if automated downscale v2 is also enabled in the environment. Signed-off-by: Peter Štibraný * CHANGELOG.md Signed-off-by: Peter Štibraný * Apply suggestions from code review Co-authored-by: Patryk Prus * Remove per-zone ingester_automated_downscale_v2 options. Signed-off-by: Peter Štibraný * Rebased, regenerate tests. Signed-off-by: Peter Štibraný --------- Signed-off-by: Peter Štibraný Co-authored-by: Patryk Prus --- CHANGELOG.md | 1 + ...test-automated-downscale-v2-generated.yaml | 2473 +++++++++++++++++ .../test-automated-downscale-v2.jsonnet | 25 + ...est-storage-ingester-autoscaling.libsonnet | 1 + .../ingester-automated-downscale-v2.libsonnet | 88 + operations/mimir/mimir.libsonnet | 1 + operations/mimir/replica-template.libsonnet | 2 +- operations/mimir/rollout-operator.libsonnet | 2 +- 8 files changed, 2591 insertions(+), 2 deletions(-) create mode 100644 operations/mimir-tests/test-automated-downscale-v2-generated.yaml create mode 100644 operations/mimir-tests/test-automated-downscale-v2.jsonnet create mode 100644 operations/mimir/ingester-automated-downscale-v2.libsonnet diff --git a/CHANGELOG.md b/CHANGELOG.md index 3aa9dffb0c..abedf0c84a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,6 +170,7 @@ * [ENHANCEMENT] Configure experimental ingest storage on query-frontend too when enabled. #8843 * [ENHANCEMENT] Allow to override Kafka client ID on a per-component basis. #9026 * [ENHANCEMENT] Rollout-operator's access to ReplicaTemplate is now configured via config option `rollout_operator_replica_template_access_enabled`. #9252 +* [ENHANCEMENT] Added support for new way of downscaling ingesters, using rollout-operator's resource-mirroring feature and read-only mode of ingesters. This can be enabled by using `ingester_automated_downscale_v2_enabled` config option. This is mutually exclusive with both `ingester_automated_downscale_enabled` (previous downscale mode) and `ingest_storage_ingester_autoscaling_enabled` (autoscaling for ingest-storage). * [BUGFIX] Added missing node affinity matchers to write component. #8910 ### Mimirtool diff --git a/operations/mimir-tests/test-automated-downscale-v2-generated.yaml b/operations/mimir-tests/test-automated-downscale-v2-generated.yaml new file mode 100644 index 0000000000..2ac5bfdced --- /dev/null +++ b/operations/mimir-tests/test-automated-downscale-v2-generated.yaml @@ -0,0 +1,2473 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: default +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: alertmanager +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: compactor +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: distributor + name: distributor + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: distributor +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: ingester-rollout + name: ingester-rollout + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + rollout-group: ingester +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached + name: memcached + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached-frontend + name: memcached-frontend + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached-frontend +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached-index-queries + name: memcached-index-queries + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached-index-queries +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: memcached-metadata + name: memcached-metadata + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: memcached-metadata +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: querier + name: querier + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: querier +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: query-frontend + name: query-frontend + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: query-frontend +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: query-scheduler + name: query-scheduler + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: query-scheduler +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: rollout-operator + name: rollout-operator + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: rollout-operator +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: ruler + name: ruler + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: ruler +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + name: store-gateway-rollout + name: store-gateway-rollout + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + rollout-group: store-gateway +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rollout-operator + namespace: default +--- +apiVersion: v1 +data: + overrides.yaml: | + overrides: {} +kind: ConfigMap +metadata: + name: overrides + namespace: default +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: replicatemplates.rollout-operator.grafana.com +spec: + group: rollout-operator.grafana.com + names: + categories: + - all + kind: ReplicaTemplate + plural: replicatemplates + singular: replicatemplate + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Status replicas + jsonPath: .status.replicas + name: StatusReplicas + type: string + - description: Spec replicas + jsonPath: .spec.replicas + name: SpecReplicas + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + labelSelector: + type: string + replicas: + default: 1 + minimum: 0 + type: integer + type: object + status: + properties: + replicas: + type: integer + type: object + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .spec.labelSelector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rollout-operator-role + namespace: default +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - watch + - delete +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - list + - get + - watch + - patch +- apiGroups: + - apps + resources: + - statefulsets/status + verbs: + - update +- apiGroups: + - rollout-operator.grafana.com + resources: + - replicatemplates/scale + - replicatemplates/status + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rollout-operator-rolebinding + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rollout-operator-role +subjects: +- kind: ServiceAccount + name: rollout-operator + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + clusterIP: None + ports: + - name: alertmanager-http-metrics + port: 8080 + targetPort: 8080 + - name: alertmanager-grpc + port: 9095 + targetPort: 9095 + - name: alertmanager-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: alertmanager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: distributor + name: distributor + namespace: default +spec: + clusterIP: None + ports: + - name: distributor-http-metrics + port: 8080 + targetPort: 8080 + - name: distributor-grpc + port: 9095 + targetPort: 9095 + - name: distributor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: distributor +--- +apiVersion: v1 +kind: Service +metadata: + name: gossip-ring + namespace: default +spec: + clusterIP: None + ports: + - appProtocol: tcp + name: gossip-ring + port: 7946 + protocol: TCP + targetPort: 7946 + selector: + gossip_ring_member: "true" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ingester-zone-a + name: ingester-zone-a + namespace: default +spec: + clusterIP: None + ports: + - name: ingester-http-metrics + port: 8080 + targetPort: 8080 + - name: ingester-grpc + port: 9095 + targetPort: 9095 + - name: ingester-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: ingester-zone-a + rollout-group: ingester +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ingester-zone-b + name: ingester-zone-b + namespace: default +spec: + clusterIP: None + ports: + - name: ingester-http-metrics + port: 8080 + targetPort: 8080 + - name: ingester-grpc + port: 9095 + targetPort: 9095 + - name: ingester-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: ingester-zone-b + rollout-group: ingester +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ingester-zone-c + name: ingester-zone-c + namespace: default +spec: + clusterIP: None + ports: + - name: ingester-http-metrics + port: 8080 + targetPort: 8080 + - name: ingester-grpc + port: 9095 + targetPort: 9095 + - name: ingester-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: ingester-zone-c + rollout-group: ingester +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached + name: memcached + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-frontend + name: memcached-frontend + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-frontend +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-index-queries + name: memcached-index-queries + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-index-queries +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-metadata + name: memcached-metadata + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-metadata +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: querier + name: querier + namespace: default +spec: + ports: + - name: querier-http-metrics + port: 8080 + targetPort: 8080 + - name: querier-grpc + port: 9095 + targetPort: 9095 + - name: querier-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: querier +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-frontend + name: query-frontend + namespace: default +spec: + ports: + - name: query-frontend-http-metrics + port: 8080 + targetPort: 8080 + - name: query-frontend-grpc + port: 9095 + targetPort: 9095 + selector: + name: query-frontend +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-scheduler + name: query-scheduler + namespace: default +spec: + ports: + - name: query-scheduler-http-metrics + port: 8080 + targetPort: 8080 + - name: query-scheduler-grpc + port: 9095 + targetPort: 9095 + selector: + name: query-scheduler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-scheduler + name: query-scheduler-discovery + namespace: default +spec: + clusterIP: None + ports: + - name: query-scheduler-http-metrics + port: 8080 + targetPort: 8080 + - name: query-scheduler-grpc + port: 9095 + targetPort: 9095 + publishNotReadyAddresses: true + selector: + name: query-scheduler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ruler + name: ruler + namespace: default +spec: + ports: + - name: ruler-http-metrics + port: 8080 + targetPort: 8080 + - name: ruler-grpc + port: 9095 + targetPort: 9095 + selector: + name: ruler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: store-gateway-multi-zone + name: store-gateway-multi-zone + namespace: default +spec: + ports: + - name: store-gateway-http-metrics + port: 80 + protocol: TCP + targetPort: 80 + selector: + rollout-group: store-gateway +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: store-gateway-zone-a + name: store-gateway-zone-a + namespace: default +spec: + clusterIP: None + ports: + - name: store-gateway-http-metrics + port: 8080 + targetPort: 8080 + - name: store-gateway-grpc + port: 9095 + targetPort: 9095 + - name: store-gateway-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: store-gateway-zone-a + rollout-group: store-gateway +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: store-gateway-zone-b + name: store-gateway-zone-b + namespace: default +spec: + clusterIP: None + ports: + - name: store-gateway-http-metrics + port: 8080 + targetPort: 8080 + - name: store-gateway-grpc + port: 9095 + targetPort: 9095 + - name: store-gateway-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: store-gateway-zone-b + rollout-group: store-gateway +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: store-gateway-zone-c + name: store-gateway-zone-c + namespace: default +spec: + clusterIP: None + ports: + - name: store-gateway-http-metrics + port: 8080 + targetPort: 8080 + - name: store-gateway-grpc + port: 9095 + targetPort: 9095 + - name: store-gateway-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: store-gateway-zone-c + rollout-group: store-gateway +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: distributor + namespace: default +spec: + minReadySeconds: 10 + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: distributor + strategy: + rollingUpdate: + maxSurge: 15% + maxUnavailable: 0 + template: + metadata: + labels: + gossip_ring_member: "true" + name: distributor + spec: + containers: + - args: + - -distributor.ha-tracker.enable=true + - -distributor.ha-tracker.enable-for-all-users=true + - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 + - -distributor.ha-tracker.prefix=prom_ha/ + - -distributor.ha-tracker.store=etcd + - -distributor.health-check-ingesters=true + - -distributor.ingestion-burst-size=200000 + - -distributor.ingestion-rate-limit=10000 + - -distributor.ring.heartbeat-period=1m + - -distributor.ring.heartbeat-timeout=4m + - -distributor.ring.prefix= + - -distributor.ring.store=memberlist + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -ingester.ring.zone-awareness-enabled=true + - -mem-ballast-size-bytes=1073741824 + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.max-connection-age=60s + - -server.grpc.keepalive.max-connection-age-grace=5m + - -server.grpc.keepalive.max-connection-idle=1m + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -shutdown-delay=90s + - -target=distributor + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "8" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: distributor + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 4Gi + requests: + cpu: "2" + memory: 2Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 100 + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: distributor + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: querier + namespace: default +spec: + minReadySeconds: 10 + replicas: 6 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: querier + strategy: + rollingUpdate: + maxSurge: 15% + maxUnavailable: 0 + template: + metadata: + labels: + gossip_ring_member: "true" + name: querier + spec: + containers: + - args: + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -ingester.ring.zone-awareness-enabled=true + - -mem-ballast-size-bytes=268435456 + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -querier.frontend-client.grpc-max-send-msg-size=104857600 + - -querier.max-concurrent=8 + - -querier.max-partial-query-length=768h + - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local.:9095 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.prefix=multi-zone/ + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.sharding-ring.zone-awareness-enabled=true + - -target=querier + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "5" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "5000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: querier + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 24Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 180 + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: querier + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: query-frontend + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: query-frontend + strategy: + rollingUpdate: + maxSurge: 15% + maxUnavailable: 0 + template: + metadata: + labels: + name: query-frontend + spec: + containers: + - args: + - -query-frontend.cache-results=true + - -query-frontend.max-cache-freshness=10m + - -query-frontend.max-total-query-length=12000h + - -query-frontend.query-sharding-target-series-per-shard=2500 + - -query-frontend.results-cache.backend=memcached + - -query-frontend.results-cache.memcached.addresses=dnssrvnoa+memcached-frontend.default.svc.cluster.local.:11211 + - -query-frontend.results-cache.memcached.max-item-size=5242880 + - -query-frontend.results-cache.memcached.timeout=500ms + - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local.:9095 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.max-connection-age=30s + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -shutdown-delay=90s + - -target=query-frontend + - -usage-stats.installation-mode=jsonnet + env: + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "5000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: query-frontend + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 1200Mi + requests: + cpu: "2" + memory: 600Mi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 390 + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: query-frontend + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: query-scheduler + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: query-scheduler + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + name: query-scheduler + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: query-scheduler + topologyKey: kubernetes.io/hostname + containers: + - args: + - -query-scheduler.max-outstanding-requests-per-tenant=100 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=query-scheduler + - -usage-stats.installation-mode=jsonnet + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: query-scheduler + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 2Gi + requests: + cpu: "2" + memory: 1Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 180 + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rollout-operator + namespace: default +spec: + minReadySeconds: 10 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: rollout-operator + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + name: rollout-operator + spec: + containers: + - args: + - -kubernetes.namespace=default + image: grafana/rollout-operator:v0.17.0 + imagePullPolicy: IfNotPresent + name: rollout-operator + ports: + - containerPort: 8001 + name: http-metrics + readinessProbe: + httpGet: + path: /ready + port: 8001 + initialDelaySeconds: 5 + timeoutSeconds: 1 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + serviceAccountName: rollout-operator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ruler + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ruler + strategy: + rollingUpdate: + maxSurge: 50% + maxUnavailable: 0 + template: + metadata: + labels: + gossip_ring_member: "true" + name: ruler + spec: + containers: + - args: + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -distributor.remote-timeout=10s + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -ingester.ring.zone-awareness-enabled=true + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -querier.max-partial-query-length=768h + - -ruler-storage.cache.backend=memcached + - -ruler-storage.cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -ruler-storage.cache.memcached.max-async-concurrency=50 + - -ruler-storage.cache.memcached.max-item-size=1048576 + - -ruler-storage.gcs.bucket-name=rules-bucket + - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local./alertmanager + - -ruler.max-rule-groups-per-tenant=70 + - -ruler.max-rules-per-rule-group=20 + - -ruler.ring.store=memberlist + - -ruler.rule-path=/rules + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.prefix=multi-zone/ + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.sharding-ring.zone-awareness-enabled=true + - -target=ruler + - -usage-stats.installation-mode=jsonnet + env: + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: ruler + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + cpu: "16" + memory: 16Gi + requests: + cpu: "1" + memory: 6Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 600 + topologySpreadConstraints: + - labelSelector: + matchLabels: + name: ruler + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: alertmanager + serviceName: alertmanager + template: + metadata: + labels: + gossip_ring_member: "true" + name: alertmanager + spec: + containers: + - args: + - -alertmanager-storage.gcs.bucket-name=alerts-bucket + - -alertmanager.sharding-ring.replication-factor=3 + - -alertmanager.sharding-ring.store=memberlist + - -alertmanager.storage.path=/data + - -alertmanager.web.external-url=http://test/alertmanager + - -common.storage.backend=gcs + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-idle-timeout=6m + - -server.http-listen-port=8080 + - -target=alertmanager + - -usage-stats.installation-mode=jsonnet + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: alertmanager + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 15Gi + requests: + cpu: "2" + memory: 10Gi + volumeMounts: + - mountPath: /data + name: alertmanager-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 900 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: alertmanager-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + name: compactor + serviceName: compactor + template: + metadata: + labels: + gossip_ring_member: "true" + name: compactor + spec: + containers: + - args: + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -compactor.block-ranges=2h,12h,24h + - -compactor.blocks-retention-period=0 + - -compactor.cleanup-interval=15m + - -compactor.compaction-concurrency=1 + - -compactor.compaction-interval=30m + - -compactor.compactor-tenant-shard-size=1 + - -compactor.data-dir=/data + - -compactor.deletion-delay=2h + - -compactor.first-level-compaction-wait-period=25m + - -compactor.max-closing-blocks-concurrency=2 + - -compactor.max-opening-blocks-concurrency=4 + - -compactor.ring.heartbeat-period=1m + - -compactor.ring.heartbeat-timeout=4m + - -compactor.ring.prefix= + - -compactor.ring.store=memberlist + - -compactor.ring.wait-stability-min-duration=1m + - -compactor.split-and-merge-shards=0 + - -compactor.split-groups=1 + - -compactor.symbols-flushers-concurrency=4 + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=compactor + - -usage-stats.installation-mode=jsonnet + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: compactor + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 6Gi + requests: + cpu: 1 + memory: 6Gi + volumeMounts: + - mountPath: /data + name: compactor-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 900 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: compactor-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 250Gi + storageClassName: standard +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown + grafana.com/prepare-downscale-http-port: "80" + grafana.com/rollout-delayed-downscale: 13h + grafana.com/rollout-mirror-replicas-from-resource-api-version: rollout-operator.grafana.com/v1 + grafana.com/rollout-mirror-replicas-from-resource-kind: ReplicaTemplate + grafana.com/rollout-mirror-replicas-from-resource-name: ingester-zone-a + grafana.com/rollout-prepare-delayed-downscale-url: http://pod/ingester/prepare-instance-ring-downscale + rollout-max-unavailable: "50" + labels: + grafana.com/min-time-between-zones-downscale: "0" + grafana.com/prepare-downscale: "true" + rollout-group: ingester + name: ingester-zone-a + namespace: default +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + name: ingester-zone-a + rollout-group: ingester + serviceName: ingester-zone-a + template: + metadata: + labels: + gossip_ring_member: "true" + name: ingester-zone-a + rollout-group: ingester + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - ingester + - key: name + operator: NotIn + values: + - ingester-zone-a + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -blocks-storage.tsdb.block-ranges-period=2h + - -blocks-storage.tsdb.dir=/data/tsdb + - -blocks-storage.tsdb.head-compaction-interval=15m + - -blocks-storage.tsdb.ship-interval=1m + - -blocks-storage.tsdb.wal-replay-concurrency=3 + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -ingester.max-global-metadata-per-metric=10 + - -ingester.max-global-metadata-per-user=30000 + - -ingester.max-global-series-per-user=150000 + - -ingester.ring.heartbeat-period=2m + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.instance-availability-zone=zone-a + - -ingester.ring.num-tokens=512 + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -ingester.ring.tokens-file-path=/data/tokens + - -ingester.ring.unregister-on-shutdown=true + - -ingester.ring.zone-awareness-enabled=true + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc-max-concurrent-streams=500 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=ingester + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "9" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: ingester + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 25Gi + requests: + cpu: "4" + memory: 15Gi + volumeMounts: + - mountPath: /data + name: ingester-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 1200 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: OnDelete + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ingester-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: fast +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown + grafana.com/prepare-downscale-http-port: "80" + grafana.com/rollout-delayed-downscale: 13h + grafana.com/rollout-mirror-replicas-from-resource-api-version: rollout-operator.grafana.com/v1 + grafana.com/rollout-mirror-replicas-from-resource-kind: ReplicaTemplate + grafana.com/rollout-mirror-replicas-from-resource-name: ingester-zone-a + grafana.com/rollout-mirror-replicas-from-resource-write-back: "false" + grafana.com/rollout-prepare-delayed-downscale-url: http://pod/ingester/prepare-instance-ring-downscale + rollout-max-unavailable: "50" + labels: + grafana.com/min-time-between-zones-downscale: "0" + grafana.com/prepare-downscale: "true" + rollout-group: ingester + name: ingester-zone-b + namespace: default +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + name: ingester-zone-b + rollout-group: ingester + serviceName: ingester-zone-b + template: + metadata: + labels: + gossip_ring_member: "true" + name: ingester-zone-b + rollout-group: ingester + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - ingester + - key: name + operator: NotIn + values: + - ingester-zone-b + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -blocks-storage.tsdb.block-ranges-period=2h + - -blocks-storage.tsdb.dir=/data/tsdb + - -blocks-storage.tsdb.head-compaction-interval=15m + - -blocks-storage.tsdb.ship-interval=1m + - -blocks-storage.tsdb.wal-replay-concurrency=3 + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -ingester.max-global-metadata-per-metric=10 + - -ingester.max-global-metadata-per-user=30000 + - -ingester.max-global-series-per-user=150000 + - -ingester.ring.heartbeat-period=2m + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.instance-availability-zone=zone-b + - -ingester.ring.num-tokens=512 + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -ingester.ring.tokens-file-path=/data/tokens + - -ingester.ring.unregister-on-shutdown=true + - -ingester.ring.zone-awareness-enabled=true + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc-max-concurrent-streams=500 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=ingester + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "9" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: ingester + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 25Gi + requests: + cpu: "4" + memory: 15Gi + volumeMounts: + - mountPath: /data + name: ingester-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 1200 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: OnDelete + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ingester-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: fast +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown + grafana.com/prepare-downscale-http-port: "80" + grafana.com/rollout-delayed-downscale: 13h + grafana.com/rollout-mirror-replicas-from-resource-api-version: rollout-operator.grafana.com/v1 + grafana.com/rollout-mirror-replicas-from-resource-kind: ReplicaTemplate + grafana.com/rollout-mirror-replicas-from-resource-name: ingester-zone-a + grafana.com/rollout-mirror-replicas-from-resource-write-back: "false" + grafana.com/rollout-prepare-delayed-downscale-url: http://pod/ingester/prepare-instance-ring-downscale + rollout-max-unavailable: "50" + labels: + grafana.com/min-time-between-zones-downscale: "0" + grafana.com/prepare-downscale: "true" + rollout-group: ingester + name: ingester-zone-c + namespace: default +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + name: ingester-zone-c + rollout-group: ingester + serviceName: ingester-zone-c + template: + metadata: + labels: + gossip_ring_member: "true" + name: ingester-zone-c + rollout-group: ingester + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - ingester + - key: name + operator: NotIn + values: + - ingester-zone-c + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -blocks-storage.tsdb.block-ranges-period=2h + - -blocks-storage.tsdb.dir=/data/tsdb + - -blocks-storage.tsdb.head-compaction-interval=15m + - -blocks-storage.tsdb.ship-interval=1m + - -blocks-storage.tsdb.wal-replay-concurrency=3 + - -common.storage.backend=gcs + - -distributor.health-check-ingesters=true + - -ingester.max-global-metadata-per-metric=10 + - -ingester.max-global-metadata-per-user=30000 + - -ingester.max-global-series-per-user=150000 + - -ingester.ring.heartbeat-period=2m + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.instance-availability-zone=zone-c + - -ingester.ring.num-tokens=512 + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=memberlist + - -ingester.ring.tokens-file-path=/data/tokens + - -ingester.ring.unregister-on-shutdown=true + - -ingester.ring.zone-awareness-enabled=true + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc-max-concurrent-streams=500 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=ingester + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "9" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: ingester + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 25Gi + requests: + cpu: "4" + memory: 15Gi + volumeMounts: + - mountPath: /data + name: ingester-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 1200 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: OnDelete + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ingester-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: fast +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached + serviceName: memcached + template: + metadata: + labels: + name: memcached + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 6144 + - -I 1m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.28-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 9Gi + requests: + cpu: 500m + memory: 6552Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.4 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-frontend + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached-frontend + serviceName: memcached-frontend + template: + metadata: + labels: + name: memcached-frontend + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-frontend + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 1024 + - -I 5m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.28-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 1536Mi + requests: + cpu: 500m + memory: 1176Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.4 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-index-queries + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached-index-queries + serviceName: memcached-index-queries + template: + metadata: + labels: + name: memcached-index-queries + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-index-queries + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 1024 + - -I 5m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.28-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 1536Mi + requests: + cpu: 500m + memory: 1176Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.4 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-metadata + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + name: memcached-metadata + serviceName: memcached-metadata + template: + metadata: + labels: + name: memcached-metadata + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-metadata + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 512 + - -I 1m + - -c 16384 + - -v + - --extended=track_sizes + image: memcached:1.6.28-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 768Mi + requests: + cpu: 500m + memory: 638Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.14.4 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + grafana.com/prepare-downscale-http-path: store-gateway/prepare-shutdown + grafana.com/prepare-downscale-http-port: "80" + rollout-max-unavailable: "50" + labels: + grafana.com/min-time-between-zones-downscale: 15m + grafana.com/prepare-downscale: "true" + rollout-group: store-gateway + name: store-gateway-zone-a + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + name: store-gateway-zone-a + rollout-group: store-gateway + serviceName: store-gateway-zone-a + template: + metadata: + labels: + gossip_ring_member: "true" + name: store-gateway-zone-a + rollout-group: store-gateway + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - store-gateway + - key: name + operator: NotIn + values: + - store-gateway-zone-a + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.bucket-store.chunks-cache.backend=memcached + - -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dnssrvnoa+memcached.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.index-cache.backend=memcached + - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.index-cache.memcached.max-item-size=5242880 + - -blocks-storage.bucket-store.index-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.auto-forget-enabled=false + - -store-gateway.sharding-ring.heartbeat-period=1m + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.instance-availability-zone=zone-a + - -store-gateway.sharding-ring.prefix=multi-zone/ + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.sharding-ring.tokens-file-path=/data/tokens + - -store-gateway.sharding-ring.unregister-on-shutdown=false + - -store-gateway.sharding-ring.wait-stability-min-duration=1m + - -store-gateway.sharding-ring.zone-awareness-enabled=true + - -target=store-gateway + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "5" + - name: GOMEMLIMIT + value: "12884901888" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: store-gateway + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 18Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /data + name: store-gateway-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: OnDelete + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: store-gateway-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: standard +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + grafana.com/prepare-downscale-http-path: store-gateway/prepare-shutdown + grafana.com/prepare-downscale-http-port: "80" + grafana.com/rollout-downscale-leader: store-gateway-zone-a + rollout-max-unavailable: "50" + labels: + grafana.com/min-time-between-zones-downscale: 15m + grafana.com/prepare-downscale: "true" + rollout-group: store-gateway + name: store-gateway-zone-b + namespace: default +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + name: store-gateway-zone-b + rollout-group: store-gateway + serviceName: store-gateway-zone-b + template: + metadata: + labels: + gossip_ring_member: "true" + name: store-gateway-zone-b + rollout-group: store-gateway + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - store-gateway + - key: name + operator: NotIn + values: + - store-gateway-zone-b + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.bucket-store.chunks-cache.backend=memcached + - -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dnssrvnoa+memcached.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.index-cache.backend=memcached + - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.index-cache.memcached.max-item-size=5242880 + - -blocks-storage.bucket-store.index-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.auto-forget-enabled=false + - -store-gateway.sharding-ring.heartbeat-period=1m + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.instance-availability-zone=zone-b + - -store-gateway.sharding-ring.prefix=multi-zone/ + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.sharding-ring.tokens-file-path=/data/tokens + - -store-gateway.sharding-ring.unregister-on-shutdown=false + - -store-gateway.sharding-ring.wait-stability-min-duration=1m + - -store-gateway.sharding-ring.zone-awareness-enabled=true + - -target=store-gateway + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "5" + - name: GOMEMLIMIT + value: "12884901888" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: store-gateway + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 18Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /data + name: store-gateway-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: OnDelete + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: store-gateway-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: standard +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + grafana.com/prepare-downscale-http-path: store-gateway/prepare-shutdown + grafana.com/prepare-downscale-http-port: "80" + grafana.com/rollout-downscale-leader: store-gateway-zone-b + rollout-max-unavailable: "50" + labels: + grafana.com/min-time-between-zones-downscale: 15m + grafana.com/prepare-downscale: "true" + rollout-group: store-gateway + name: store-gateway-zone-c + namespace: default +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + name: store-gateway-zone-c + rollout-group: store-gateway + serviceName: store-gateway-zone-c + template: + metadata: + labels: + gossip_ring_member: "true" + name: store-gateway-zone-c + rollout-group: store-gateway + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rollout-group + operator: In + values: + - store-gateway + - key: name + operator: NotIn + values: + - store-gateway-zone-c + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.bucket-store.chunks-cache.backend=memcached + - -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dnssrvnoa+memcached.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.index-cache.backend=memcached + - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.index-cache.memcached.max-item-size=5242880 + - -blocks-storage.bucket-store.index-cache.memcached.timeout=750ms + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local.:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections=150 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -common.storage.backend=gcs + - -memberlist.bind-port=7946 + - -memberlist.join=dns+gossip-ring.default.svc.cluster.local.:7946 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.auto-forget-enabled=false + - -store-gateway.sharding-ring.heartbeat-period=1m + - -store-gateway.sharding-ring.heartbeat-timeout=4m + - -store-gateway.sharding-ring.instance-availability-zone=zone-c + - -store-gateway.sharding-ring.prefix=multi-zone/ + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=memberlist + - -store-gateway.sharding-ring.tokens-file-path=/data/tokens + - -store-gateway.sharding-ring.unregister-on-shutdown=false + - -store-gateway.sharding-ring.wait-stability-min-duration=1m + - -store-gateway.sharding-ring.zone-awareness-enabled=true + - -target=store-gateway + - -usage-stats.installation-mode=jsonnet + env: + - name: GOMAXPROCS + value: "5" + - name: GOMEMLIMIT + value: "12884901888" + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1000" + image: grafana/mimir:2.13.0 + imagePullPolicy: IfNotPresent + name: store-gateway + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + - containerPort: 7946 + name: gossip-ring + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 18Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /data + name: store-gateway-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: OnDelete + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: store-gateway-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: standard +--- +apiVersion: etcd.database.coreos.com/v1beta2 +kind: EtcdCluster +metadata: + annotations: + etcd.database.coreos.com/scope: clusterwide + name: etcd + namespace: default +spec: + pod: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + etcd_cluster: etcd + topologyKey: kubernetes.io/hostname + annotations: + prometheus.io/port: "2379" + prometheus.io/scrape: "true" + etcdEnv: + - name: ETCD_AUTO_COMPACTION_RETENTION + value: 1h + labels: + name: etcd + resources: + limits: + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + size: 3 + version: 3.3.13 +--- +apiVersion: rollout-operator.grafana.com/v1 +kind: ReplicaTemplate +metadata: + name: ingester-zone-a + namespace: default +spec: + labelSelector: name=ingester-zone-a + replicas: 1 diff --git a/operations/mimir-tests/test-automated-downscale-v2.jsonnet b/operations/mimir-tests/test-automated-downscale-v2.jsonnet new file mode 100644 index 0000000000..a438c3374f --- /dev/null +++ b/operations/mimir-tests/test-automated-downscale-v2.jsonnet @@ -0,0 +1,25 @@ +local mimir = import 'mimir/mimir.libsonnet'; + +mimir { + _config+:: { + namespace: 'default', + external_url: 'http://test', + + storage_backend: 'gcs', + blocks_storage_bucket_name: 'blocks-bucket', + + ruler_enabled: true, + ruler_storage_bucket_name: 'rules-bucket', + + alertmanager_enabled: true, + alertmanager_storage_bucket_name: 'alerts-bucket', + + ingester_automated_downscale_v2_enabled: true, + multi_zone_ingester_enabled: true, + multi_zone_ingester_replicas: 3, + + store_gateway_automated_downscale_enabled: true, + multi_zone_store_gateway_enabled: true, + multi_zone_store_gateway_replicas: 3, + }, +} diff --git a/operations/mimir/ingest-storage-ingester-autoscaling.libsonnet b/operations/mimir/ingest-storage-ingester-autoscaling.libsonnet index 8536c23835..2450096ae7 100644 --- a/operations/mimir/ingest-storage-ingester-autoscaling.libsonnet +++ b/operations/mimir/ingest-storage-ingester-autoscaling.libsonnet @@ -29,6 +29,7 @@ assert !$._config.ingest_storage_ingester_autoscaling_enabled || $._config.multi_zone_ingester_enabled : 'partitions ingester autoscaling is only supported with multi-zone ingesters in namespace %s' % $._config.namespace, assert !$._config.ingest_storage_ingester_autoscaling_ingester_annotations_enabled || $._config.multi_zone_ingester_replicas <= 0 : 'partitions ingester autoscaling and multi_zone_ingester_replicas are mutually exclusive in namespace %s' % $._config.namespace, assert !$._config.ingest_storage_ingester_autoscaling_ingester_annotations_enabled || !$._config.ingester_automated_downscale_enabled : 'partitions ingester autoscaling and ingester automated downscale config are mutually exclusive in namespace %s' % $._config.namespace, + assert !$._config.ingest_storage_ingester_autoscaling_ingester_annotations_enabled || !$._config.ingester_automated_downscale_v2_enabled : 'partitions ingester autoscaling and ingester automated downscale v2 config are mutually exclusive in namespace %s' % $._config.namespace, assert !$._config.ingest_storage_ingester_autoscaling_enabled || $.rollout_operator_deployment != null : 'partitions ingester autoscaling requires rollout-operator in namespace %s' % $._config.namespace, // Create resource that will be targetted by ScaledObject. diff --git a/operations/mimir/ingester-automated-downscale-v2.libsonnet b/operations/mimir/ingester-automated-downscale-v2.libsonnet new file mode 100644 index 0000000000..3dddf79bb9 --- /dev/null +++ b/operations/mimir/ingester-automated-downscale-v2.libsonnet @@ -0,0 +1,88 @@ +{ + // Allow ingesters to be safely downscaled using the "prepare-downscale" and "resource-mirroring" features + // of the rollout-operator. Only available when using multi-zone ingesters. This feature can be enabled + // via the ingester_automated_downscale_v2_enabled flag. + // + // When enabled, ReplicaTemplate/ingester-zone-a object is installed into the namespace. This object holds desired + // number of replicas for each ingester zone in spec.replicas field. (There is only one ReplicaTemplate object, + // and all ingester zones use it.) + // + // Scaledown is initiated by decreasing spec.replicas field of ReplicaTemplate/ingester-zone-a object. + // Rollout-operator will notice the change, set ingesters that are going to be downscaled (in all zones) + // to read-only mode, and wait until ingesters were in read-only mode for pre-configured time + // (ingester_automated_downscale_delay). + // + // After this time is reached, rollout-operator will finally adjust number of replicas in ingester StatefulSets + // (for all zones). + // + // The number of replicas is still controlled by $._config.multi_zone_ingester_replicas until a reasonable metric to scale it + // automatically is determined and tested. + _config+: { + ingester_automated_downscale_v2_enabled: false, + + // Number of ingester replicas per zone. All zones will have the same number of replicas. + ingester_automated_downscale_v2_ingester_zones: 3, + ingester_automated_downscale_v2_replicas_per_zone: std.ceil($._config.multi_zone_ingester_replicas / $._config.ingester_automated_downscale_v2_ingester_zones), + + // How long to wait before terminating an ingester after it has been notified about the scale down. + ingester_automated_downscale_v2_delay: if 'querier.query-ingesters-within' in $.querier_args then + $.querier_args['querier.query-ingesters-within'] + else + // The default -querier.query-ingesters-within in Mimir is 13 hours. + '13h', + }, + + // Validate the configuration. + assert !$._config.ingester_automated_downscale_v2_enabled || $._config.multi_zone_ingester_enabled : 'ingester downscaling requires multi_zone_ingester_enabled in namespace %s' % $._config.namespace, + assert !$._config.ingester_automated_downscale_v2_enabled || $._config.multi_zone_ingester_replicas > 0 : 'ingester downscaling requires multi_zone_ingester_replicas > 0 in namespace %s' % $._config.namespace, + assert !$._config.ingester_automated_downscale_v2_enabled || !$._config.ingester_automated_downscale_enabled : 'ingester_automated_downscale_enabled_v2 and ingester_automated_downscale_enabled are mutually exclusive in namespace %s' % $._config.namespace, + + // Utility used to override a field only if exists in super. + local overrideSuperIfExists(name, override) = if !( name in super) || super[name] == null || super[name] == {} then null else + super[name] + override, + + local updateIngesterZone(enabled, statefulsetName, primary=true) = overrideSuperIfExists( + statefulsetName, + local statefulSet = $.apps.v1.statefulSet; + + if !enabled then {} else ( + $.removeReplicasFromSpec + + statefulSet.mixin.metadata.withLabelsMixin({ + // Let the rollout-operator know that it must call the prepare-shutdown endpoint before + // scaling down ingesters in this statefulset. + 'grafana.com/prepare-downscale': 'true', + // Zones are scaled down at the same time, with no delay between the zones. + 'grafana.com/min-time-between-zones-downscale': '0', + }) + + statefulSet.mixin.metadata.withAnnotationsMixin({ + // Endpoint for telling ingester that it's going to be scaled down. Ingester will flush the + // data and unregister from the ring on shutdown. + 'grafana.com/prepare-downscale-http-path': 'ingester/prepare-shutdown', + 'grafana.com/prepare-downscale-http-port': '80', + // Ingester statefulset will follow number of replicas from ReplicaTemplate/ingester-zone-a. + 'grafana.com/rollout-mirror-replicas-from-resource-api-version': 'rollout-operator.grafana.com/v1', + 'grafana.com/rollout-mirror-replicas-from-resource-kind': 'ReplicaTemplate', + 'grafana.com/rollout-mirror-replicas-from-resource-name': 'ingester-zone-a', + 'grafana.com/rollout-delayed-downscale': $._config.ingester_automated_downscale_v2_delay, + 'grafana.com/rollout-prepare-delayed-downscale-url': 'http://pod/ingester/prepare-instance-ring-downscale', + } + ( + if !primary then { + // Disable updates of ReplicaTemplate/ingester-zone-a from non-primary (zone-b, zone-c) statefulsets. + 'grafana.com/rollout-mirror-replicas-from-resource-write-back': 'false', + } + else {} + )) + ) + ), + + // Create resource that will be targetted by ScaledObject. A single ReplicaTemplate is used for all zones. + // HPA requires that label selector exists and is valid, but it will not be used for target type of AverageValue. + // In GKE however we see that selector is used to find pods and compute current usage, so we set it to target pods with given name. + ingester_zone_a_replica_template: + if !$._config.ingester_automated_downscale_v2_enabled then null + else $.replicaTemplate('ingester-zone-a', $._config.ingester_automated_downscale_v2_replicas_per_zone, 'name=ingester-zone-a'), + + ingester_zone_a_statefulset: updateIngesterZone($._config.ingester_automated_downscale_v2_enabled, 'ingester_zone_a_statefulset', true), + ingester_zone_b_statefulset: updateIngesterZone($._config.ingester_automated_downscale_v2_enabled, 'ingester_zone_b_statefulset', false), + ingester_zone_c_statefulset: updateIngesterZone($._config.ingester_automated_downscale_v2_enabled, 'ingester_zone_c_statefulset', false), +} diff --git a/operations/mimir/mimir.libsonnet b/operations/mimir/mimir.libsonnet index 845b2edb00..9042f40fea 100644 --- a/operations/mimir/mimir.libsonnet +++ b/operations/mimir/mimir.libsonnet @@ -40,6 +40,7 @@ // Automated downscale of ingesters and store-gateways (import 'ingester-automated-downscale.libsonnet') + +(import 'ingester-automated-downscale-v2.libsonnet') + (import 'store-gateway-automated-downscale.libsonnet') + // Automatic cleanup of unused PVCs after scaling down diff --git a/operations/mimir/replica-template.libsonnet b/operations/mimir/replica-template.libsonnet index 687a115f8f..1b848d16d5 100644 --- a/operations/mimir/replica-template.libsonnet +++ b/operations/mimir/replica-template.libsonnet @@ -1,6 +1,6 @@ { _config+: { - replica_template_custom_resource_definition_enabled: $._config.ingest_storage_ingester_autoscaling_enabled, + replica_template_custom_resource_definition_enabled: $._config.ingest_storage_ingester_autoscaling_enabled || $._config.ingester_automated_downscale_v2_enabled, }, replica_template:: std.parseYaml(importstr 'replica-templates.yaml'), diff --git a/operations/mimir/rollout-operator.libsonnet b/operations/mimir/rollout-operator.libsonnet index 5d8ae4c715..1e381dfb8b 100644 --- a/operations/mimir/rollout-operator.libsonnet +++ b/operations/mimir/rollout-operator.libsonnet @@ -30,7 +30,7 @@ // Ignore these labels used for controlling webhook behavior when creating services. service_ignored_labels+:: ['grafana.com/no-downscale', 'grafana.com/prepare-downscale'], - rollout_operator_replica_template_access_enabled: $._config.ingest_storage_ingester_autoscaling_enabled, + rollout_operator_replica_template_access_enabled: $._config.ingest_storage_ingester_autoscaling_enabled || $._config.ingester_automated_downscale_v2_enabled, }, local rollout_operator_enabled =