From dfcf7d61d9e86cdc6c3ebaafee67469cef158c81 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 16 May 2023 17:24:51 -0700 Subject: [PATCH] helm: enable clusterapi namespace autodiscovery --- charts/cluster-autoscaler/Chart.yaml | 2 +- charts/cluster-autoscaler/README.md | 2 + charts/cluster-autoscaler/README.md.gotmpl | 1 + charts/cluster-autoscaler/templates/NOTES.txt | 4 +- .../cluster-autoscaler/templates/_helpers.tpl | 48 +++++++++++++++---- .../templates/deployment.yaml | 4 +- charts/cluster-autoscaler/values.yaml | 3 ++ .../cloudprovider/clusterapi/README.md | 5 +- 8 files changed, 50 insertions(+), 19 deletions(-) diff --git a/charts/cluster-autoscaler/Chart.yaml b/charts/cluster-autoscaler/Chart.yaml index 80b88c6df67..e58e4b197ba 100644 --- a/charts/cluster-autoscaler/Chart.yaml +++ b/charts/cluster-autoscaler/Chart.yaml @@ -11,4 +11,4 @@ name: cluster-autoscaler sources: - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler type: application -version: 9.35.0 +version: 9.36.0 diff --git a/charts/cluster-autoscaler/README.md b/charts/cluster-autoscaler/README.md index f7aba27dc3a..fb6ab2b47b0 100644 --- a/charts/cluster-autoscaler/README.md +++ b/charts/cluster-autoscaler/README.md @@ -222,6 +222,7 @@ $ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml `cloudProvider: clusterapi` must be set, and then one or more of - `autoDiscovery.clusterName` +- or `autoDiscovery.namespace` - or `autoDiscovery.labels` See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details. @@ -373,6 +374,7 @@ vpa: | affinity | object | `{}` | Affinity for pod assignment | | autoDiscovery.clusterName | string | `nil` | Enable autodiscovery for `cloudProvider=aws`, for groups matching `autoDiscovery.tags`. autoDiscovery.clusterName -- Enable autodiscovery for `cloudProvider=azure`, using tags defined in https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md#auto-discovery-setup. Enable autodiscovery for `cloudProvider=clusterapi`, for groups matching `autoDiscovery.labels`. Enable autodiscovery for `cloudProvider=gce`, but no MIG tagging required. Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. | | autoDiscovery.labels | list | `[]` | Cluster-API labels to match https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery | +| autoDiscovery.namespace | string | `nil` | Enable autodiscovery via cluster namespace for for `cloudProvider=clusterapi` | | autoDiscovery.roles | list | `["worker"]` | Magnum node group roles to match. | | autoDiscovery.tags | list | `["k8s.io/cluster-autoscaler/enabled","k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }}"]` | ASG tags to match, run through `tpl`. | | autoscalingGroups | list | `[]` | For AWS, Azure AKS or Magnum. At least one element is required if not using `autoDiscovery`. For example: 
 - name: asg1
   maxSize: 2
   minSize: 1
| diff --git a/charts/cluster-autoscaler/README.md.gotmpl b/charts/cluster-autoscaler/README.md.gotmpl index c91d0d57244..c098aac18bd 100644 --- a/charts/cluster-autoscaler/README.md.gotmpl +++ b/charts/cluster-autoscaler/README.md.gotmpl @@ -222,6 +222,7 @@ $ helm install my-release autoscaler/cluster-autoscaler -f myvalues.yaml `cloudProvider: clusterapi` must be set, and then one or more of - `autoDiscovery.clusterName` +- or `autoDiscovery.namespace` - or `autoDiscovery.labels` See [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md#configuring-node-group-auto-discovery) for more details. diff --git a/charts/cluster-autoscaler/templates/NOTES.txt b/charts/cluster-autoscaler/templates/NOTES.txt index 94e211e96b1..1a87a3d10b5 100644 --- a/charts/cluster-autoscaler/templates/NOTES.txt +++ b/charts/cluster-autoscaler/templates/NOTES.txt @@ -1,4 +1,4 @@ -{{- if or .Values.autoDiscovery.clusterName .Values.autoscalingGroups -}} +{{- if or ( or .Values.autoDiscovery.clusterName .Values.autoDiscovery.namespace .Values.autoDiscovery.labels ) .Values.autoscalingGroups }} To verify that cluster-autoscaler has started, run: @@ -8,7 +8,7 @@ To verify that cluster-autoscaler has started, run: ############################################################################## #### ERROR: You must specify values for either #### -#### autoDiscovery.clusterName or autoscalingGroups[] #### +#### autoDiscovery or autoscalingGroups[] #### ############################################################################## The deployment and pod will not be created and the installation is not functional diff --git a/charts/cluster-autoscaler/templates/_helpers.tpl b/charts/cluster-autoscaler/templates/_helpers.tpl index b0aee4fc080..c7e80f4d8e4 100644 --- a/charts/cluster-autoscaler/templates/_helpers.tpl +++ b/charts/cluster-autoscaler/templates/_helpers.tpl @@ -40,14 +40,11 @@ app.kubernetes.io/name: {{ include "cluster-autoscaler.name" . | quote }} {{/* -Return labels, including instance, name and version. +Return labels, including instance and name. */}} {{- define "cluster-autoscaler.labels" -}} {{ include "cluster-autoscaler.instance-name" . }} app.kubernetes.io/managed-by: {{ .Release.Service | quote }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} helm.sh/chart: {{ include "cluster-autoscaler.chart" . | quote }} {{- if .Values.additionalLabels }} {{ toYaml .Values.additionalLabels }} @@ -112,16 +109,23 @@ Return true if the priority expander is enabled {{- end -}} {{/* -Return the autodiscoveryparameters for clusterapi. +autoDiscovery.clusterName for clusterapi. */}} -{{- define "cluster-autoscaler.capiAutodiscoveryConfig" -}} -{{- if .Values.autoDiscovery.clusterName -}} +{{- define "cluster-autoscaler.capiAutodiscovery.clusterName" -}} {{- print "clusterName=" -}}{{ tpl (.Values.autoDiscovery.clusterName) . }} {{- end -}} -{{- if and .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels -}} -{{- print "," -}} + +{{/* +autoDiscovery.namespace for clusterapi. +*/}} +{{- define "cluster-autoscaler.capiAutodiscovery.namespace" -}} +{{- print "namespace=" }}{{ .Values.autoDiscovery.namespace -}} {{- end -}} -{{- if .Values.autoDiscovery.labels -}} + +{{/* +autoDiscovery.labels for clusterapi. +*/}} +{{- define "cluster-autoscaler.capiAutodiscovery.labels" -}} {{- range $i, $el := .Values.autoDiscovery.labels -}} {{- if $i -}}{{- print "," -}}{{- end -}} {{- range $key, $val := $el -}} @@ -129,4 +133,28 @@ Return the autodiscoveryparameters for clusterapi. {{- end -}} {{- end -}} {{- end -}} + +{{/* +Return the autodiscoveryparameters for clusterapi. +*/}} +{{- define "cluster-autoscaler.capiAutodiscoveryConfig" -}} +{{- if .Values.autoDiscovery.clusterName -}} +{{ include "cluster-autoscaler.capiAutodiscovery.clusterName" . }} + {{- if .Values.autoDiscovery.namespace }} + {{- print "," -}} + {{ include "cluster-autoscaler.capiAutodiscovery.namespace" . }} + {{- end -}} + {{- if .Values.autoDiscovery.labels }} + {{- print "," -}} + {{ include "cluster-autoscaler.capiAutodiscovery.labels" . }} + {{- end -}} +{{- else if .Values.autoDiscovery.namespace -}} +{{ include "cluster-autoscaler.capiAutodiscovery.namespace" . }} + {{- if .Values.autoDiscovery.labels }} + {{- print "," -}} + {{ include "cluster-autoscaler.capiAutodiscovery.labels" . }} + {{- end -}} +{{- else if .Values.autoDiscovery.labels -}} + {{ include "cluster-autoscaler.capiAutodiscovery.labels" . }} +{{- end -}} {{- end -}} diff --git a/charts/cluster-autoscaler/templates/deployment.yaml b/charts/cluster-autoscaler/templates/deployment.yaml index ccbe4353edc..2ba43d6c7ee 100644 --- a/charts/cluster-autoscaler/templates/deployment.yaml +++ b/charts/cluster-autoscaler/templates/deployment.yaml @@ -1,4 +1,4 @@ -{{- if or ( or .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels ) .Values.autoscalingGroups }} +{{- if or ( or .Values.autoDiscovery.clusterName .Values.autoDiscovery.namespace .Values.autoDiscovery.labels ) .Values.autoscalingGroups }} {{/* one of the above is required */}} apiVersion: {{ template "deployment.apiVersion" . }} kind: Deployment @@ -95,7 +95,7 @@ spec: - --cluster-name={{ tpl (.Values.magnumClusterName) . }} {{- end }} {{- else if eq .Values.cloudProvider "clusterapi" }} - {{- if or .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels }} + {{- if or .Values.autoDiscovery.clusterName .Values.autoDiscovery.labels .Values.autoDiscovery.namepace }} - --node-group-auto-discovery=clusterapi:{{ template "cluster-autoscaler.capiAutodiscoveryConfig" . }} {{- end }} {{- if eq .Values.clusterAPIMode "incluster-kubeconfig"}} diff --git a/charts/cluster-autoscaler/values.yaml b/charts/cluster-autoscaler/values.yaml index 0affb0242d3..703333951ea 100644 --- a/charts/cluster-autoscaler/values.yaml +++ b/charts/cluster-autoscaler/values.yaml @@ -16,6 +16,9 @@ autoDiscovery: # Enable autodiscovery for `cloudProvider=magnum`, for groups matching `autoDiscovery.roles`. clusterName: # cluster.local + # autoDiscovery.namespace -- Enable autodiscovery via cluster namespace for for `cloudProvider=clusterapi` + namespace: # default + # autoDiscovery.tags -- ASG tags to match, run through `tpl`. tags: - k8s.io/cluster-autoscaler/enabled diff --git a/cluster-autoscaler/cloudprovider/clusterapi/README.md b/cluster-autoscaler/cloudprovider/clusterapi/README.md index 2f4e247f457..61232616210 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/README.md +++ b/cluster-autoscaler/cloudprovider/clusterapi/README.md @@ -50,9 +50,7 @@ most likely need other command line flags. For more information you can invoke ## Configuring node group auto discovery -If you do not configure node group auto discovery, cluster autoscaler will attempt -to match nodes against any scalable resources found in any namespace and belonging -to any Cluster. +You must configure node group auto discovery to inform cluster autoscaler which cluster in which to find for scalable node groups. Limiting cluster autoscaler to only match against resources in the blue namespace @@ -454,4 +452,3 @@ here as a reference for users who might be deploying on these infrastructures. | IBM Cloud | `ibm-cloud.kubernetes.io/worker-id` | Used by the IBM Cloud Cloud Controller Manager to identify the node | | IBM Cloud | `vpc-block-csi-driver-labels` | Used by the IBM Cloud CSI driver as a target for persistent volume node affinity | | IBM Cloud | `ibm-cloud.kubernetes.io/vpc-instance-id` | Used when a VPC is in use on IBM Cloud | -