From 7438c4e8c456a8e2811e8933e46fefc85edc3ebb Mon Sep 17 00:00:00 2001 From: Aleksei Sizov Date: Mon, 26 Aug 2024 16:14:18 -0500 Subject: [PATCH] Add Azure hosted CP template --- docs/azure/hosted-control-plane.md | 162 +++++++++++++ templates/azure-hosted-cp/.helmignore | 23 ++ templates/azure-hosted-cp/Chart.yaml | 20 ++ .../azure-hosted-cp/templates/_helpers.tpl | 43 ++++ .../templates/azurecluster.yaml | 31 +++ .../templates/azuremachinetemplate.yaml | 20 ++ .../azure-hosted-cp/templates/cluster.yaml | 17 ++ .../templates/k0smotroncontrolplane.yaml | 49 ++++ .../templates/k0sworkerconfigtemplate.yaml | 15 ++ .../templates/machinedeployment.yaml | 26 ++ templates/azure-hosted-cp/values.schema.json | 228 ++++++++++++++++++ templates/azure-hosted-cp/values.yaml | 55 +++++ .../files/templates/azure-hosted-cp.yaml | 8 + 13 files changed, 697 insertions(+) create mode 100644 docs/azure/hosted-control-plane.md create mode 100644 templates/azure-hosted-cp/.helmignore create mode 100644 templates/azure-hosted-cp/Chart.yaml create mode 100644 templates/azure-hosted-cp/templates/_helpers.tpl create mode 100644 templates/azure-hosted-cp/templates/azurecluster.yaml create mode 100644 templates/azure-hosted-cp/templates/azuremachinetemplate.yaml create mode 100644 templates/azure-hosted-cp/templates/cluster.yaml create mode 100644 templates/azure-hosted-cp/templates/k0smotroncontrolplane.yaml create mode 100644 templates/azure-hosted-cp/templates/k0sworkerconfigtemplate.yaml create mode 100644 templates/azure-hosted-cp/templates/machinedeployment.yaml create mode 100644 templates/azure-hosted-cp/values.schema.json create mode 100644 templates/azure-hosted-cp/values.yaml create mode 100644 templates/hmc-templates/files/templates/azure-hosted-cp.yaml diff --git a/docs/azure/hosted-control-plane.md b/docs/azure/hosted-control-plane.md new file mode 100644 index 000000000..14329e74f --- /dev/null +++ b/docs/azure/hosted-control-plane.md @@ -0,0 +1,162 @@ +# Hosted control plane (k0smotron) deployment + +## Prerequisites + +- Management Kubernetes cluster (v1.28+) deployed on Azure with HMC installed + on it +- Default storage class configured on the management cluster + +Keep in mind that all control plane components for all managed clusters will +reside in the management cluster. + +## Pre-existing resources + +Certain resources will not be created automatically in a hosted control plane +scenario thus they should be created in advance and provided in the `Deployment` +object. You can reuse these resources with management cluster as described +below. + +If you deployed your Azure Kubernetes cluster using Cluster API Provider Azure +(CAPZ) you can obtain all the necessary data with the commands below: + +**Location** + +```bash +kubectl get azurecluster -o go-template='{{.spec.location}}' +``` + +**Subscription ID** + +```bash +kubectl get azurecluster -o go-template='{{.spec.subscriptionID}}' +``` + +**Resource group** + +```bash +kubectl get azurecluster -o go-template='{{.spec.resourceGroup}}' +``` + +**vnet name** + +```bash +kubectl get azurecluster -o go-template='{{.spec.networkSpec.vnet.name}}' +``` + +**Subnet name** + +```bash +kubectl get azurecluster -o go-template='{{(index .spec.networkSpec.subnets 1).name}}' +``` + +**Route table name** + +```bash +kubectl get azurecluster -o go-template='{{(index .spec.networkSpec.subnets 1).routeTable.name}}' +``` + +**Security group name** + +```bash +kubectl get azurecluster -o go-template='{{(index .spec.networkSpec.subnets 1).securityGroup.name}}' +``` + + + +## HMC Deployment manifest + +With all the collected data your `Deployment` manifest will look similar to this: + +```yaml +apiVersion: hmc.mirantis.com/v1alpha1 +kind: Deployment +metadata: + name: azure-hosted-cp +spec: + template: azure-hosted-cp + config: + location: "westus" + subscriptionID: ceb131c7-a917-439f-8e19-cd59fe247e03 + vmSize: Standard_A4_v2 + clusterIdentity: + name: az-cluster-identity + namespace: hmc-system + resourceGroup: mgmt-cluster + network: + vnetName: mgmt-cluster-vnet + nodeSubnetName: mgmt-cluster-node-subnet + routeTableName: mgmt-cluster-node-routetable + securityGroupName: mgmt-cluster-node-nsg + tenantID: 7db9e0f2-c88a-4116-a373-9c8b6cc9d5eb + clientID: 471f65fa-ddee-40b4-90ae-da1a8a114ee1 + clientSecret: "u_RANDOM" +``` + +To simplify creation of the deployment object you can use the template below: + +```yaml +apiVersion: hmc.mirantis.com/v1alpha1 +kind: Deployment +metadata: + name: azure-hosted-cp +spec: + template: azure-hosted-cp + config: + location: "{{.spec.location}}" + subscriptionID: "{{.spec.subscriptionID}}" + vmSize: Standard_A4_v2 + clusterIdentity: + name: az-cluster-identity + namespace: hmc-system + resourceGroup: "{{.spec.resourceGroup}}" + network: + vnetName: "{{.spec.networkSpec.vnet.name}}" + nodeSubnetName: "{{(index .spec.networkSpec.subnets 1).name}}" + routeTableName: "{{(index .spec.networkSpec.subnets 1).routeTable.name}}" + securityGroupName: "{{(index .spec.networkSpec.subnets 1).securityGroup.name}}" + tenantID: 7db9e0f2-c88a-4116-a373-9c8b6cc9d5eb + clientID: 471f65fa-ddee-40b4-90ae-da1a8a114ee1 + clientSecret: "u_RANDOM" +``` + +Then you can render it using the command: + +```bash +kubectl get azurecluster -o go-template="$(cat template.yaml)" +``` + +## Cluster creation + +After applying `Deployment` object you require to manually set the status of the +`AzureCluster` object due to current limitations (see k0sproject/k0smotron#668). + +To do so you need to execute the following command: + +```bash +kubectl patch azurecluster --type=merge --subresource status --patch 'status: {ready: true}' +``` + +## Important notes on the cluster deletion + +Because of the aforementioned limitation you also need to make manual steps in +order to properly delete cluster. + +Before removing the cluster make sure to place custom finalizer onto +`AzureCluster` object. This is needed to prevent it from being deleted instantly +which will cause cluster deletion to stuck indefinitely. + +To place finalizer you can execute the following command: + +```bash +kubectl patch azurecluster --type=merge --patch 'metadata: {finalizers: [manual]}' +``` + +When finalizer is placed you can remove the `Deployment` as usual. Check that +all `AzureMachines` objects are deleted successfully and remove finalizer you've +placed to finish cluster deletion. + +In case if have orphaned `AzureMachines` left you have to delete finalizers on +them manually after making sure that no VMs are present in Azure. + +*Note: since Azure admission prohibits orphaned objects mutation you'll have to +disable it by deleting it's `mutatingwebhookconfiguration`* diff --git a/templates/azure-hosted-cp/.helmignore b/templates/azure-hosted-cp/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/templates/azure-hosted-cp/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/templates/azure-hosted-cp/Chart.yaml b/templates/azure-hosted-cp/Chart.yaml new file mode 100644 index 000000000..5c3bfcbf6 --- /dev/null +++ b/templates/azure-hosted-cp/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v2 +name: azure-hosted-cp +description: | + An HMC template to deploy a k8s cluster on Azure with control plane components + within the management cluster. +type: application +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.30.4+k0s.0" +annotations: + hmc.mirantis.com/type: deployment + hmc.mirantis.com/infrastructure-providers: azure + hmc.mirantis.com/controlplane-providers: k0s + hmc.mirantis.com/bootstrap-providers: k0s diff --git a/templates/azure-hosted-cp/templates/_helpers.tpl b/templates/azure-hosted-cp/templates/_helpers.tpl new file mode 100644 index 000000000..dae9addfb --- /dev/null +++ b/templates/azure-hosted-cp/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{- define "cluster.name" -}} + {{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "azuremachinetemplate.name" -}} + {{- include "cluster.name" . }}-mt +{{- end }} + +{{- define "k0smotroncontrolplane.name" -}} + {{- include "cluster.name" . }}-cp +{{- end }} + +{{- define "k0sworkerconfigtemplate.name" -}} + {{- include "cluster.name" . }}-machine-config +{{- end }} + +{{- define "machinedeployment.name" -}} + {{- include "cluster.name" . }}-md +{{- end }} + +{{- define "azure.json" -}} +{ + "cloud": "AzurePublicCloud", + "tenantId": "{{ .Values.tenantID }}", + "subscriptionId": "{{ .Values.subscriptionID }}", + "aadClientId": "{{ .Values.clientID }}", + "aadClientSecret": "{{ .Values.clientSecret }}", + "resourceGroup": "{{ .Values.resourceGroup }}", + "securityGroupName": "{{ .Values.network.securityGroupName }}", + "securityGroupResourceGroup": "{{ .Values.resourceGroup }}", + "location": "{{ .Values.location }}", + "vmType": "vmss", + "vnetName": "{{ .Values.network.vnetName }}", + "vnetResourceGroup": "{{ .Values.resourceGroup }}", + "subnetName": "{{ .Values.network.nodeSubnetName }}", + "routeTableName": "{{ .Values.routeTableName }}", + "loadBalancerSku": "Standard", + "loadBalancerName": "", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true +} +{{- end }} diff --git a/templates/azure-hosted-cp/templates/azurecluster.yaml b/templates/azure-hosted-cp/templates/azurecluster.yaml new file mode 100644 index 000000000..07cb15845 --- /dev/null +++ b/templates/azure-hosted-cp/templates/azurecluster.yaml @@ -0,0 +1,31 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: {{ include "cluster.name" . }} + annotations: + cluster.x-k8s.io/managed-by: k0smotron +spec: + identityRef: + kind: AzureClusterIdentity + name: {{ .Values.clusterIdentity.name }} + namespace: {{ .Values.clusterIdentity.namespace }} + networkSpec: + vnet: + resourceGroup: {{ .Values.resourceGroup }} + name: {{ .Values.network.vnetName }} + subnets: + - name: {{ .Values.network.nodeSubnetName }} + role: node + routeTable: + name: {{ .Values.network.routeTableName }} + securityGroup: + name: {{ .Values.network.securityGroupName }} + location: {{ .Values.location }} + {{- if .Values.bastion.enabled }} + {{- with .Values.bastion.bastionSpec }} + bastionSpec: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + subscriptionID: {{ .Values.subscriptionID }} + resourceGroup: {{ .Values.resourceGroup }} diff --git a/templates/azure-hosted-cp/templates/azuremachinetemplate.yaml b/templates/azure-hosted-cp/templates/azuremachinetemplate.yaml new file mode 100644 index 000000000..9725b36a8 --- /dev/null +++ b/templates/azure-hosted-cp/templates/azuremachinetemplate.yaml @@ -0,0 +1,20 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: {{ include "azuremachinetemplate.name" . }} +spec: + template: + spec: + osDisk: + diskSizeGB: {{ .Values.rootVolumeSize }} + osType: Linux + {{- if not ( .Values.sshPublicKey | empty) }} + sshPublicKey: {{ .Values.sshPublicKey }} + {{- end }} + vmSize: {{ .Values.vmSize }} + {{- if not (quote .Values.image | empty) }} + {{- with .Values.image }} + image: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} diff --git a/templates/azure-hosted-cp/templates/cluster.yaml b/templates/azure-hosted-cp/templates/cluster.yaml new file mode 100644 index 000000000..74bd07a54 --- /dev/null +++ b/templates/azure-hosted-cp/templates/cluster.yaml @@ -0,0 +1,17 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: {{ include "cluster.name" . }} +spec: + {{- with .Values.clusterNetwork }} + clusterNetwork: + {{- toYaml . | nindent 4 }} + {{- end }} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: K0smotronControlPlane + name: {{ include "k0smotroncontrolplane.name" . }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: {{ include "cluster.name" . }} diff --git a/templates/azure-hosted-cp/templates/k0smotroncontrolplane.yaml b/templates/azure-hosted-cp/templates/k0smotroncontrolplane.yaml new file mode 100644 index 000000000..c353a085b --- /dev/null +++ b/templates/azure-hosted-cp/templates/k0smotroncontrolplane.yaml @@ -0,0 +1,49 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: K0smotronControlPlane +metadata: + name: {{ include "k0smotroncontrolplane.name" . }} +spec: + replicas: {{ .Values.controlPlaneNumber }} + version: {{ .Values.k0s.version | replace "+" "-" }} + {{- with .Values.k0smotron.service }} + service: + {{- toYaml . | nindent 4 }} + {{- end }} + controllerPlaneFlags: + - "--enable-cloud-provider=true" + - "--debug=true" + k0sConfig: + apiVersion: k0s.k0sproject.io/v1beta1 + kind: ClusterConfig + metadata: + name: k0s + spec: + network: + provider: calico + calico: + mode: vxlan + extensions: + helm: + repositories: + - name: cloud-provider-azure + url: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + - name: azuredisk-csi-driver + url: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + charts: + - name: cloud-provider-azure + namespace: kube-system + chartname: cloud-provider-azure/cloud-provider-azure + version: 1.30.4 + order: 1 + values: | + cloudControllerManager: + nodeSelector: + node-role.kubernetes.io/control-plane: null + - name: azuredisk-csi-driver + namespace: kube-system + chartname: azuredisk-csi-driver/azuredisk-csi-driver + version: 1.30.3 + order: 2 + values: | + linux: + kubelet: "/var/lib/k0s/kubelet" diff --git a/templates/azure-hosted-cp/templates/k0sworkerconfigtemplate.yaml b/templates/azure-hosted-cp/templates/k0sworkerconfigtemplate.yaml new file mode 100644 index 000000000..1233d640e --- /dev/null +++ b/templates/azure-hosted-cp/templates/k0sworkerconfigtemplate.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: K0sWorkerConfigTemplate +metadata: + name: {{ include "k0sworkerconfigtemplate.name" . }} +spec: + template: + spec: + version: {{ .Values.k0s.version }} + args: + - --enable-cloud-provider + - --kubelet-extra-args="--cloud-provider=external" + files: + - path: "/etc/kubernetes/azure.json" + permissions: "0644" + content: {{ include "azure.json" . | toJson }} diff --git a/templates/azure-hosted-cp/templates/machinedeployment.yaml b/templates/azure-hosted-cp/templates/machinedeployment.yaml new file mode 100644 index 000000000..e5995a1fb --- /dev/null +++ b/templates/azure-hosted-cp/templates/machinedeployment.yaml @@ -0,0 +1,26 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: {{ include "machinedeployment.name" . }} +spec: + clusterName: {{ include "cluster.name" . }} + replicas: {{ .Values.workersNumber }} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }} + spec: + version: {{ regexReplaceAll "\\+k0s.+$" .Values.k0s.version "" }} + clusterName: {{ include "cluster.name" . }} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: K0sWorkerConfigTemplate + name: {{ include "k0sworkerconfigtemplate.name" . }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: {{ include "azuremachinetemplate.name" . }} diff --git a/templates/azure-hosted-cp/values.schema.json b/templates/azure-hosted-cp/values.schema.json new file mode 100644 index 000000000..9d6b4e391 --- /dev/null +++ b/templates/azure-hosted-cp/values.schema.json @@ -0,0 +1,228 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "An HMC template to deploy a K8s cluster on Azure with control plane components within the management cluster.", + "type": "object", + "required": [ + "controlPlaneNumber", + "workersNumber", + "location", + "subscriptionID", + "tenantID", + "clientID", + "clientSecret", + "clusterIdentity", + "resourceGroup", + "network", + "vmSize" + ], + "properties": { + "controlPlaneNumber": { + "description": "The number of the control plane pods", + "type": "number", + "minimum": 1 + }, + "workersNumber": { + "description": "The number of the worker machines", + "type": "number", + "minimum": 1 + }, + "clusterNetwork": { + "type": "object", + "properties": { + "pods": { + "type": "object", + "properties": { + "cidrBlocks": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + } + }, + "services": { + "type": "object", + "properties": { + "cidrBlocks": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + } + } + } + }, + "location": { + "description": "Azure location to deploy the cluster in", + "type": "string" + }, + "subscriptionID": { + "description": "Azure subscription ID which will be used for all resources", + "type": "string" + }, + "tenantID": { + "description": "Tenant ID for the service principal", + "type": "string" + }, + "clientID": { + "description": "Client ID of the service principal", + "type": "string" + }, + "clientSecret": { + "description": "Client secret of the service principal", + "type": "string" + }, + "bastion": { + "type": "object", + "description": "The configuration of the bastion host", + "required": [], + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "clusterIdentity": { + "type": "object", + "description": "AzureClusterIdentity object reference", + "required": [ + "name", + "namespace" + ], + "properties": { + "name": { + "description": "AzureClusterIdentity object name", + "type": "string" + }, + "namespace": { + "description": "AzureClusterIdentity object namespace", + "type": "string" + } + } + }, + "resourceGroup": { + "description": "Existing resource group name for worker nodes", + "type": "string" + }, + "network": { + "type": "object", + "description": "Networking resources reference for worker nodes", + "required": [ + "vnetName", + "nodeSubnetName", + "routeTableName", + "securityGroupName" + ], + "properties": { + "vnetName": { + "description": "Existing vnet name for worker nodes", + "type": "string" + }, + "nodeSubnetName": { + "description": "Existing subnet name for worker nodes", + "type": "string" + }, + "routeTableName": { + "description": "Existing route table name for worker nodes", + "type": "string" + }, + "securityGroupName": { + "description": "Existing security group name for worker nodes", + "type": "string" + } + } + }, + "sshPublicKey": { + "description": "SSH public key in base64 format, which will be used on the machine.", + "type": "string" + }, + "vmSize": { + "description": "The size of instance to create", + "type": "string" + }, + "rootVolumeSize": { + "description": "The size of the root volume of the instance (GB)", + "type": "integer" + }, + "image": { + "type": "object", + "description": "Azure VM image configuration", + "properties": { + "marketplace": { + "description": "Azure Marketplace image reference", + "type": "object", + "required": [ + "publisher", + "offer", + "sku", + "version" + ], + "properties": { + "publisher": { + "type": "string" + }, + "offer": { + "type": "string" + }, + "sku": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + } + }, + "k0smotron": { + "description": "K0smotron parameters", + "type": "object", + "properties": { + "service": { + "description": "The configuration of a K0smotron service", + "properties": { + "type": { + "description": "Ingress methods for a k0smotron service", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ], + "type": "string" + }, + "apiPort": { + "description": "The kubernetes API port for a k0smotron service", + "type": "number", + "minimum": 1, + "maximum": 65535 + }, + "konnectivityPort": { + "description": "The konnectivity port", + "type": "number", + "minimum": 1, + "maximum": 65535 + } + } + } + } + }, + "k0s": { + "description": "K0s parameters", + "type": "object", + "required": [ + "version" + ], + "properties": { + "version":{ + "description": "K0s version to use", + "type": "string" + } + } + } + } +} diff --git a/templates/azure-hosted-cp/values.yaml b/templates/azure-hosted-cp/values.yaml new file mode 100644 index 000000000..6feea59a5 --- /dev/null +++ b/templates/azure-hosted-cp/values.yaml @@ -0,0 +1,55 @@ +# Cluster parameters +controlPlaneNumber: 3 +workersNumber: 2 + +clusterNetwork: + pods: + cidrBlocks: + - "10.244.0.0/16" + services: + cidrBlocks: + - "10.96.0.0/12" + +# Azure cluster parameters +location: "" +subscriptionID: "" +# k0sproject/k0smotron#692 workaround +tenantID: "" +clientID: "" +clientSecret: "" +### +bastion: + enabled: false + bastionSpec: + azureBastion: {} +clusterIdentity: + name: "" + namespace: hmc-system +resourceGroup: "" +network: + vnetName: "" + nodeSubnetName: "" + routeTableName: "" + securityGroupName: "" +# Azure machines parameters + +sshPublicKey: "" +vmSize: "" +rootVolumeSize: 30 +image: + marketplace: + publisher: "cncf-upstream" + offer: "capi" + sku: "ubuntu-2204-gen1" + version: "130.3.20240717" + +# K0smotron parameters +k0smotron: + service: + type: LoadBalancer + apiPort: 6443 + konnectivityPort: 8132 + +# K0s parameters +k0s: + version: v1.30.4+k0s.0 diff --git a/templates/hmc-templates/files/templates/azure-hosted-cp.yaml b/templates/hmc-templates/files/templates/azure-hosted-cp.yaml new file mode 100644 index 000000000..67daba869 --- /dev/null +++ b/templates/hmc-templates/files/templates/azure-hosted-cp.yaml @@ -0,0 +1,8 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: Template +metadata: + name: azure-hosted-cp +spec: + helm: + chartName: azure-hosted-cp + chartVersion: 0.0.1