helm: set GOMAXPROCS on ingesters (#9283)

* Helm: set GOMAXPROCS on ingesters Signed-off-by: Dimitar Dimitrov <[email protected]> * Add changelog entry Signed-off-by: Dimitar Dimitrov <[email protected]> * Update CHANGELOG.md entry Signed-off-by: Dimitar Dimitrov <[email protected]> * update helm tests * Update operations/helm/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml Co-authored-by: Vernon Miller <[email protected]> --------- Signed-off-by: Dimitar Dimitrov <[email protected]> Co-authored-by: Vernon Miller <[email protected]>
grafana · Sep 17, 2024 · bdd443b · bdd443b
1 parent b616a99
commit bdd443b
Show file tree

Hide file tree

Showing 27 changed files with 199 additions and 16 deletions.
diff --git a/operations/compare-helm-with-jsonnet/helm/00-base/values.yaml b/operations/compare-helm-with-jsonnet/helm/00-base/values.yaml
@@ -22,6 +22,11 @@ rollout_operator:
 ingester:
   zoneAwareReplication:
     enabled: false
+  resources:
+    requests:
+      # request the same as jsonnet, so that the derived GOMAXPROCS are the same
+      cpu: 4
+      memory: 15Gi
 store_gateway:
   zoneAwareReplication:
     enabled: false
diff --git a/operations/compare-helm-with-jsonnet/helm/06-pods/kustomization.yaml b/operations/compare-helm-with-jsonnet/helm/06-pods/kustomization.yaml
@@ -83,11 +83,3 @@ patches:
     patch: |-
       - op: remove
         path: /spec/template/metadata/namespace
-
-  # TODO dimitarvdimitrov: GOMAXPROCS isn't present in Helm for now
-  - target:
-      name: mimir-ingester
-      kind: StatefulSet
-    patch: |-
-      - op: remove
-        path: /spec/template/spec/containers/0/env
diff --git a/operations/compare-helm-with-jsonnet/jsonnet/07-pods/kustomization.yaml b/operations/compare-helm-with-jsonnet/jsonnet/07-pods/kustomization.yaml
@@ -13,14 +13,6 @@ patches:
       - op: remove
         path: /spec/template/spec/containers/0/env
 
-  # TODO dimitarvdimitrov: GOMAXPROCS isn't present in Helm for now
-  - target:
-      name: mimir-ingester
-      kind: StatefulSet
-    patch: |-
-      - op: remove
-        path: /spec/template/spec/containers/0/env
-
   # TODO(logiraptor): Jsonnet uses a deprecated PodDisruptionBudget api version
   - target:
       name: mimir-(ingester|store-gateway)

diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md
@@ -47,6 +47,7 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [ENHANCEMENT] Helm : allow setting a read and write urls to continous-test. #8741
 * [ENHANCEMENT] Add support for running continuous-test with GEM #8837
 * [ENHANCEMENT] Alerts: `RequestErrors` and `RulerRemoteEvaluationFailing` have been enriched with a native histogram version. #9004
+* [ENHANCEMENT] Ingester: set GOMAXPROCS to help with golang scheduling overhead when running on machines with a lot of cores. #9283
 * [BUGFIX] Add missing container security context to run `continuous-test` under the restricted security policy. #8653
 * [BUGFIX] Add `global.extraVolumeMounts` to the exporter container on memcached statefulsets #8787
 * [BUGFIX] Fix helm releases failing when `querier.kedaAutoscaling.predictiveScalingEnabled=true`. #8731

diff --git a/operations/helm/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -186,6 +186,13 @@ spec:
           {{- $jaeger_queue_size := dig "jaegerReporterMaxQueueSize" nil .Values.ingester }}
           {{- if or .Values.global.extraEnv .Values.ingester.env $jaeger_queue_size }}
           env:
+            {{- $cpu_request := dig "requests" "cpu" nil .Values.ingester.resources }}
+            {{- if $cpu_request }}
+              {{- $cpu_request_between_3_and_6 := include "mimir.parseCPU" (dict "value" $cpu_request) | float64 | min 6 | max 3 }}
+            {{/* copy logic from operations/mimir/ingester.libsonnet */}}
+            - name: "GOMAXPROCS"
+              value: {{ addf $cpu_request $cpu_request_between_3_and_6 1 | ceil | toString | toYaml }}
+            {{- end }}
             {{- with .Values.global.extraEnv }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

diff --git a/...ise-https-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...ise-https-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -143,6 +143,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -290,6 +293,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -437,5 +443,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...nterprise-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...nterprise-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -134,6 +134,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -272,6 +275,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -410,5 +416,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...way-nginx-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...way-nginx-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -129,6 +129,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -262,6 +265,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -395,5 +401,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...e-enabled-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...e-enabled-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -127,5 +127,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...ng-global-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...ng-global-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -129,6 +129,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -262,6 +265,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -395,5 +401,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...onitoring-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...onitoring-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -129,6 +129,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -262,6 +265,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -395,5 +401,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...toscaling-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...toscaling-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -129,6 +129,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -262,6 +265,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -395,5 +401,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...sts/large-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...sts/large-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -145,6 +145,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "8"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -294,6 +297,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "8"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -443,5 +449,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "8"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...onitoring-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...onitoring-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -122,5 +122,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...openshift-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...openshift-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -131,6 +131,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -266,6 +269,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -401,5 +407,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...uler-name-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...uler-name-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -130,6 +130,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -264,6 +267,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -398,5 +404,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...sts/small-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...sts/small-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -145,6 +145,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "8"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -294,6 +297,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "8"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -443,5 +449,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "8"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
diff --git a/...-k8s-1.25-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...-k8s-1.25-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -134,6 +134,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -272,6 +275,9 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"
 ---
@@ -410,5 +416,8 @@ spec:
               - ALL
             readOnlyRootFilesystem: true
           env:
+
+            - name: "GOMAXPROCS"
+              value: "4"
             - name: "JAEGER_REPORTER_MAX_QUEUE_SIZE"
               value: "1000"