From 52a1174a242119f311d618e8c7e1f010553a4ffe Mon Sep 17 00:00:00 2001 From: Aleksei Sizov Date: Sat, 24 Aug 2024 15:39:49 -0500 Subject: [PATCH] Add Azure standalone CP template --- docs/azure/cluster-parameters.md | 94 ++++++++ templates/azure-standalone-cp/.helmignore | 23 ++ templates/azure-standalone-cp/Chart.yaml | 19 ++ .../templates/_helpers.tpl | 71 ++++++ .../templates/azurecluster.yaml | 17 ++ .../azuremachinetemplate-controlplane.yaml | 20 ++ .../azuremachinetemplate-worker.yaml | 20 ++ .../templates/cluster.yaml | 17 ++ .../templates/k0scontrolplane.yaml | 61 +++++ .../templates/k0sworkerconfigtemplate.yaml | 15 ++ .../templates/machinedeployment.yaml | 26 +++ .../azure-standalone-cp/values.schema.json | 221 ++++++++++++++++++ templates/azure-standalone-cp/values.yaml | 53 +++++ .../files/templates/azure-standalone-cp.yaml | 8 + 14 files changed, 665 insertions(+) create mode 100644 docs/azure/cluster-parameters.md create mode 100644 templates/azure-standalone-cp/.helmignore create mode 100644 templates/azure-standalone-cp/Chart.yaml create mode 100644 templates/azure-standalone-cp/templates/_helpers.tpl create mode 100644 templates/azure-standalone-cp/templates/azurecluster.yaml create mode 100644 templates/azure-standalone-cp/templates/azuremachinetemplate-controlplane.yaml create mode 100644 templates/azure-standalone-cp/templates/azuremachinetemplate-worker.yaml create mode 100644 templates/azure-standalone-cp/templates/cluster.yaml create mode 100644 templates/azure-standalone-cp/templates/k0scontrolplane.yaml create mode 100644 templates/azure-standalone-cp/templates/k0sworkerconfigtemplate.yaml create mode 100644 templates/azure-standalone-cp/templates/machinedeployment.yaml create mode 100644 templates/azure-standalone-cp/values.schema.json create mode 100644 templates/azure-standalone-cp/values.yaml create mode 100644 templates/hmc-templates/files/templates/azure-standalone-cp.yaml diff --git a/docs/azure/cluster-parameters.md b/docs/azure/cluster-parameters.md new file mode 100644 index 000000000..e3247cb21 --- /dev/null +++ b/docs/azure/cluster-parameters.md @@ -0,0 +1,94 @@ +# Azure cluster parameters + +## Prerequisites + +- Azure CLI installed +- `az login` command executed + +## Cluster Identity + +To provide credentials for CAPI Azure provider (CAPZ) the `AzureClusterIdentity` +resource must be created. This should be done before provisioning any clusters. + + +To create the `AzureClusterIdentity` you should first get the desired +`SubscriptionID` by executing `az account list -o table` which will return list +of subscriptions available to user. + +Then you need to create service principal which will be used by CAPZ to interact +with Azure API. To do so you need to execute the following command: + +```bash + az ad sp create-for-rbac --role contributor --scopes="/subscriptions/" +``` + +The command will return json with the credentials for the service pricipal which +will look like this: + +```json + { + "appId": "29a3a125-7848-4ce6-9be9-a4b3eecca0ff", + "displayName": "azure-cli", + "password": "u_RANDOMHASH", + "tenant": "2f10bc28-959b-481f-b094-eb043a87570a", + } +``` + +*Note: make sure to save this credentials and treat them like passwords.* + +With the data from the json you can now create the `AzureClusterIdentity` object +and it's secret. + +The objects created with the data above can look somthing like this: + +**Secret**: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: az-cluster-identity-secret +stringData: + clientSecret: u_RANDOMHASH +type: Opaque +``` + +**AzureClusterIdentity**: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: az-cluster-identity +spec: + allowedNamespaces: {} + clientID: 29a3a125-7848-4ce6-9be9-a4b3eecca0ff + clientSecret: + name: az-cluster-identity-secret + tenantID: 2f10bc28-959b-481f-b094-eb043a87570a + type: ServicePrincipal +``` + +These objects then should be referenced in the `Deployment` object in the +`.spec.config.clusterIdentity` field. + +Subcribtion ID which was used to create service principal should be the same the +same that will be used in the `.spec.config.subscriptionID` field of the +`Deployment` object. + +### Cloud controller manager note + +In order for CCM to work on Azure it requires `azure.json` file to be located on +all nodes. + +Because of a limitation (k0sproject/k0smotron#692) it's not currently possible +to automatically pass credentials to all nodes. + +To mitigate that you should pass cluster identity data once again in the +following fields of a `Deployment` object: + +- `.spec.config.tenantID` +- `.spec.config.clientID` +- `.spec.config.clientSecret` diff --git a/templates/azure-standalone-cp/.helmignore b/templates/azure-standalone-cp/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/templates/azure-standalone-cp/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/templates/azure-standalone-cp/Chart.yaml b/templates/azure-standalone-cp/Chart.yaml new file mode 100644 index 000000000..2a5408dd4 --- /dev/null +++ b/templates/azure-standalone-cp/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: azure-standalone-cp +description: | + An HMC template to deploy a k0s cluster on Azure with bootstrapped control plane nodes. +type: application +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.30.2+k0s.0" +annotations: + hmc.mirantis.com/type: deployment + hmc.mirantis.com/infrastructure-providers: azure + hmc.mirantis.com/controlplane-providers: k0s + hmc.mirantis.com/bootstrap-providers: k0s diff --git a/templates/azure-standalone-cp/templates/_helpers.tpl b/templates/azure-standalone-cp/templates/_helpers.tpl new file mode 100644 index 000000000..264bb1db0 --- /dev/null +++ b/templates/azure-standalone-cp/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{- define "cluster.name" -}} + {{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "azuremachinetemplate.controlplane.name" -}} + {{- include "cluster.name" . }}-cp-mt +{{- end }} + +{{- define "azuremachinetemplate.worker.name" -}} + {{- include "cluster.name" . }}-worker-mt +{{- end }} + +{{- define "k0scontrolplane.name" -}} + {{- include "cluster.name" . }}-cp +{{- end }} + +{{- define "k0sworkerconfigtemplate.name" -}} + {{- include "cluster.name" . }}-machine-config +{{- end }} + +{{- define "machinedeployment.name" -}} + {{- include "cluster.name" . }}-md +{{- end }} + +{{- define "azure.json.worker" -}} +{ + "cloud": "AzurePublicCloud", + "tenantId": "{{ .Values.tenantID }}", + "subscriptionId": "{{ .Values.subscriptionID }}", + "aadClientId": "{{ .Values.clientID }}", + "aadClientSecret": "{{ .Values.clientSecret }}", + "resourceGroup": "{{ include "cluster.name" . }}", + "securityGroupName": "{{ include "cluster.name" . }}-node-nsg", + "securityGroupResourceGroup": "{{ include "cluster.name" . }}", + "location": "{{ .Values.location }}", + "vmType": "vmss", + "vnetName": "{{ include "cluster.name" . }}-vnet", + "vnetResourceGroup": "{{ include "cluster.name" . }}", + "subnetName": "{{ include "cluster.name" . }}-node-subnet", + "routeTableName": "{{ include "cluster.name" . }}-node-routetable", + "loadBalancerSku": "Standard", + "loadBalancerName": "", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true +} +{{- end }} + +{{- define "azure.json.controller" -}} +{ + "cloud": "AzurePublicCloud", + "tenantId": "{{ .Values.tenantID }}", + "subscriptionId": "{{ .Values.subscriptionID }}", + "aadClientId": "{{ .Values.clientID }}", + "aadClientSecret": "{{ .Values.clientSecret }}", + "resourceGroup": "{{ include "cluster.name" . }}", + "securityGroupName": "{{ include "cluster.name" . }}-controlplane-nsg", + "securityGroupResourceGroup": "{{ include "cluster.name" . }}", + "location": "{{ .Values.location }}", + "vmType": "vmss", + "vnetName": "{{ include "cluster.name" . }}-vnet", + "vnetResourceGroup": "{{ include "cluster.name" . }}", + "subnetName": "{{ include "cluster.name" . }}-controlplane-subnet", + "routeTableName": "{{ include "cluster.name" . }}-controlplane-routetable", + "loadBalancerSku": "Standard", + "loadBalancerName": "", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true +} +{{- end }} diff --git a/templates/azure-standalone-cp/templates/azurecluster.yaml b/templates/azure-standalone-cp/templates/azurecluster.yaml new file mode 100644 index 000000000..75769b590 --- /dev/null +++ b/templates/azure-standalone-cp/templates/azurecluster.yaml @@ -0,0 +1,17 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: {{ include "cluster.name" . }} +spec: + identityRef: + kind: AzureClusterIdentity + name: {{ .Values.clusterIdentity.name }} + namespace: {{ .Values.clusterIdentity.namespace }} + location: {{ .Values.location }} + {{- if .Values.bastion.enabled }} + {{- with .Values.bastion.bastionSpec }} + bastionSpec: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + subscriptionID: {{ .Values.subscriptionID }} diff --git a/templates/azure-standalone-cp/templates/azuremachinetemplate-controlplane.yaml b/templates/azure-standalone-cp/templates/azuremachinetemplate-controlplane.yaml new file mode 100644 index 000000000..2e499dd56 --- /dev/null +++ b/templates/azure-standalone-cp/templates/azuremachinetemplate-controlplane.yaml @@ -0,0 +1,20 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: {{ include "azuremachinetemplate.controlplane.name" . }} +spec: + template: + spec: + osDisk: + diskSizeGB: {{ .Values.controlPlane.rootVolumeSize }} + osType: Linux + {{- if not (quote .Values.controlPlane.sshPublicKey | empty) }} + sshPublicKey: {{ .Values.controlPlane.sshPublicKey }} + {{- end }} + vmSize: {{ .Values.controlPlane.vmSize }} + {{- if not (quote .Values.controlPlane.image | empty) }} + {{- with .Values.controlPlane.image }} + image: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} diff --git a/templates/azure-standalone-cp/templates/azuremachinetemplate-worker.yaml b/templates/azure-standalone-cp/templates/azuremachinetemplate-worker.yaml new file mode 100644 index 000000000..365214730 --- /dev/null +++ b/templates/azure-standalone-cp/templates/azuremachinetemplate-worker.yaml @@ -0,0 +1,20 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: {{ include "azuremachinetemplate.worker.name" . }} +spec: + template: + spec: + osDisk: + diskSizeGB: {{ .Values.worker.rootVolumeSize }} + osType: Linux + {{- if not (quote .Values.worker.sshPublicKey | empty) }} + sshPublicKey: {{ .Values.worker.sshPublicKey }} + {{- end }} + vmSize: {{ .Values.worker.vmSize }} + {{- if not (quote .Values.worker.image | empty) }} + {{- with .Values.worker.image }} + image: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} diff --git a/templates/azure-standalone-cp/templates/cluster.yaml b/templates/azure-standalone-cp/templates/cluster.yaml new file mode 100644 index 000000000..2ce7581f7 --- /dev/null +++ b/templates/azure-standalone-cp/templates/cluster.yaml @@ -0,0 +1,17 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: {{ include "cluster.name" . }} +spec: + {{- with .Values.clusterNetwork }} + clusterNetwork: + {{- toYaml . | nindent 4 }} + {{- end }} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: K0sControlPlane + name: {{ include "k0scontrolplane.name" . }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: {{ include "cluster.name" . }} diff --git a/templates/azure-standalone-cp/templates/k0scontrolplane.yaml b/templates/azure-standalone-cp/templates/k0scontrolplane.yaml new file mode 100644 index 000000000..8ec5035ae --- /dev/null +++ b/templates/azure-standalone-cp/templates/k0scontrolplane.yaml @@ -0,0 +1,61 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: K0sControlPlane +metadata: + name: {{ include "k0scontrolplane.name" . }} +spec: + replicas: {{ .Values.controlPlaneNumber }} + version: {{ .Values.k0s.version }} + k0sConfigSpec: + args: + - --enable-worker + - --enable-cloud-provider + - --kubelet-extra-args="--cloud-provider=external" + - --disable-components=konnectivity-server + files: + - path: "/etc/kubernetes/azure.json" + permissions: "0644" + content: {{ include "azure.json.controller" . | toJson }} + k0s: + apiVersion: k0s.k0sproject.io/v1beta1 + kind: ClusterConfig + metadata: + name: k0s + spec: + api: + extraArgs: + anonymous-auth: "true" + network: + provider: calico + calico: + mode: vxlan + extensions: + helm: + repositories: + - name: cloud-provider-azure + url: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + - name: azuredisk-csi-driver + url: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + charts: + - name: cloud-provider-azure + namespace: kube-system + chartname: cloud-provider-azure/cloud-provider-azure + version: 1.30.4 + order: 1 + values: | + cloudControllerManager: + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + - name: azuredisk-csi-driver + namespace: kube-system + chartname: azuredisk-csi-driver/azuredisk-csi-driver + version: 1.30.3 + order: 2 + values: | + linux: + kubelet: "/var/lib/k0s/kubelet" + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: {{ include "azuremachinetemplate.controlplane.name" . }} + namespace: {{ .Release.Namespace }} diff --git a/templates/azure-standalone-cp/templates/k0sworkerconfigtemplate.yaml b/templates/azure-standalone-cp/templates/k0sworkerconfigtemplate.yaml new file mode 100644 index 000000000..51f3c614f --- /dev/null +++ b/templates/azure-standalone-cp/templates/k0sworkerconfigtemplate.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: K0sWorkerConfigTemplate +metadata: + name: {{ include "k0sworkerconfigtemplate.name" . }} +spec: + template: + spec: + version: {{ .Values.k0s.version }} + args: + - --enable-cloud-provider + - --kubelet-extra-args="--cloud-provider=external" + files: + - path: "/etc/kubernetes/azure.json" + permissions: "0644" + content: {{ include "azure.json.worker" . | toJson }} diff --git a/templates/azure-standalone-cp/templates/machinedeployment.yaml b/templates/azure-standalone-cp/templates/machinedeployment.yaml new file mode 100644 index 000000000..464ae0ceb --- /dev/null +++ b/templates/azure-standalone-cp/templates/machinedeployment.yaml @@ -0,0 +1,26 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: {{ include "machinedeployment.name" . }} +spec: + clusterName: {{ include "cluster.name" . }} + replicas: {{ .Values.workersNumber }} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }} + spec: + version: {{ regexReplaceAll "\\+k0s.+$" .Values.k0s.version "" }} + clusterName: {{ include "cluster.name" . }} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: K0sWorkerConfigTemplate + name: {{ include "k0sworkerconfigtemplate.name" . }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: {{ include "azuremachinetemplate.worker.name" . }} diff --git a/templates/azure-standalone-cp/values.schema.json b/templates/azure-standalone-cp/values.schema.json new file mode 100644 index 000000000..75bf5c400 --- /dev/null +++ b/templates/azure-standalone-cp/values.schema.json @@ -0,0 +1,221 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "An HMC template to deploy a k0s cluster on Azure with bootstrapped control plane nodes.", + "type": "object", + "required": [ + "controlPlaneNumber", + "workersNumber", + "location", + "subscriptionID", + "tenantID", + "clientID", + "clientSecret", + "clusterIdentity" + ], + "properties": { + "controlPlaneNumber": { + "description": "The number of the control plane machines", + "type": "number", + "minimum": 1 + }, + "workersNumber": { + "description": "The number of the worker machines", + "type": "number", + "minimum": 1 + }, + "clusterNetwork": { + "type": "object", + "properties": { + "pods": { + "type": "object", + "properties": { + "cidrBlocks": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + } + }, + "services": { + "type": "object", + "properties": { + "cidrBlocks": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + } + } + } + }, + "location": { + "description": "Azure location to deploy the cluster in", + "type": "string" + }, + "subscriptionID": { + "description": "Azure subscription ID which will be used for all resources", + "type": "string" + }, + "tenantID": { + "description": "Tenant ID for the service principal", + "type": "string" + }, + "clientID": { + "description": "Client ID of the service principal", + "type": "string" + }, + "clientSecret": { + "description": "Client secret of the service principal", + "type": "string" + }, + "bastion": { + "type": "object", + "description": "The configuration of the bastion host", + "required": [], + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "clusterIdentity": { + "type": "object", + "description": "AzureClusterIdentity object reference", + "required": [ + "name", + "namespace" + ], + "properties": { + "name": { + "description": "AzureClusterIdentity object name", + "type": "string" + }, + "namespace": { + "description": "AzureClusterIdentity object namespace", + "type": "string" + } + } + }, + "controlPlane": { + "description": "The configuration of the control plane machines", + "type": "object", + "required": [ + "vmSize" + ], + "properties": { + "sshPublicKey": { + "description": "SSH public key in base64 format, which will be used on the machine.", + "type": "string" + }, + "vmSize": { + "description": "The size of instance to create", + "type": "string" + }, + "rootVolumeSize": { + "description": "The size of the root volume of the instance (GB)", + "type": "integer" + }, + "image": { + "type": "object", + "description": "Azure VM image configuration", + "properties": { + "marketplace": { + "description": "Azure Marketplace image reference", + "type": "object", + "required": [ + "publisher", + "offer", + "sku", + "version" + ], + "properties": { + "publisher": { + "type": "string" + }, + "offer": { + "type": "string" + }, + "sku": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + } + } + } + }, + "worker": { + "description": "The configuration of the worker machines", + "type": "object", + "required": [ + "vmSize" + ], + "properties": { + "sshPublicKey": { + "description": "SSH public key in base64 format, which will be used on the machine.", + "type": "string" + }, + "vmSize": { + "description": "The size of instance to create", + "type": "string" + }, + "rootVolumeSize": { + "description": "The size of the root volume of the instance (GB)", + "type": "integer" + }, + "image": { + "type": "object", + "description": "Azure VM image configuration", + "properties": { + "marketplace": { + "description": "Azure Marketplace image reference", + "type": "object", + "required": [ + "publisher", + "offer", + "sku", + "version" + ], + "properties": { + "publisher": { + "type": "string" + }, + "offer": { + "type": "string" + }, + "sku": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + } + } + } + }, + "k0s": { + "description": "K0s parameters", + "type": "object", + "required": [ + "version" + ], + "properties": { + "version":{ + "description": "K0s version to use", + "type": "string" + } + } + } + } +} diff --git a/templates/azure-standalone-cp/values.yaml b/templates/azure-standalone-cp/values.yaml new file mode 100644 index 000000000..089aa6b10 --- /dev/null +++ b/templates/azure-standalone-cp/values.yaml @@ -0,0 +1,53 @@ +# Cluster parameters +controlPlaneNumber: 3 +workersNumber: 2 + +clusterNetwork: + pods: + cidrBlocks: + - "10.244.0.0/16" + services: + cidrBlocks: + - "10.96.0.0/12" + +# AWS cluster parameters +location: "" +subscriptionID: "" +# k0sproject/k0smotron#692 workaround +tenantID: "" +clientID: "" +clientSecret: "" +### +bastion: + enabled: false + bastionSpec: + azureBastion: {} +clusterIdentity: + name: "" + namespace: hmc-system +# AWS machines parameters +controlPlane: + sshPublicKey: "" + vmSize: "" + rootVolumeSize: 30 + image: + marketplace: + publisher: "cncf-upstream" + offer: "capi" + sku: "ubuntu-2204-gen1" + version: "130.3.20240717" + +worker: + sshPublicKey: "" + vmSize: "" + rootVolumeSize: 30 + image: + marketplace: + publisher: "cncf-upstream" + offer: "capi" + sku: "ubuntu-2204-gen1" + version: "130.3.20240717" + +# K0s parameters +k0s: + version: v1.30.4+k0s.0 diff --git a/templates/hmc-templates/files/templates/azure-standalone-cp.yaml b/templates/hmc-templates/files/templates/azure-standalone-cp.yaml new file mode 100644 index 000000000..a8bd9856f --- /dev/null +++ b/templates/hmc-templates/files/templates/azure-standalone-cp.yaml @@ -0,0 +1,8 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: Template +metadata: + name: azure-standalone-cp +spec: + helm: + chartName: azure-standalone-cp + chartVersion: 0.0.1