From 182c7f976a6557df4d0447fd2606cee63278b828 Mon Sep 17 00:00:00 2001 From: Evan Date: Tue, 30 Apr 2024 12:51:04 -0400 Subject: [PATCH] refactor kubeadm and clusterclass flavor to use inline configuration (#296) --- .gitignore | 2 +- hack/manifests/kubeadm.yaml | 4 + .../common-init-files/kustomization.yaml | 4 - templates/common-init-files/secret.yaml | 44 ---------- .../clusterclass-base/kustomization.yaml | 6 -- .../linodeMachineControlPlaneTemplate.yaml | 13 --- .../linodeMachineWorkerTemplate.yaml | 13 --- .../deleteTransformer.yaml | 80 +++++++++++++++++++ .../kubeadmConfigTemplate.yaml | 39 --------- .../clusterclass-kubeadm/kustomization.yaml | 51 ++++++++++-- .../linodeClusterTemplate.yaml | 0 .../replacementTransformer.yaml | 30 +++++++ .../default/kubeadmConfigTemplate.yaml | 53 ++++++++---- .../flavors/default/kubeadmControlPlane.yaml | 53 ++++++++---- templates/flavors/default/kustomization.yaml | 1 - 15 files changed, 234 insertions(+), 159 deletions(-) delete mode 100644 templates/common-init-files/kustomization.yaml delete mode 100644 templates/common-init-files/secret.yaml delete mode 100644 templates/flavors/clusterclass-base/kustomization.yaml delete mode 100644 templates/flavors/clusterclass-base/linodeMachineControlPlaneTemplate.yaml delete mode 100644 templates/flavors/clusterclass-base/linodeMachineWorkerTemplate.yaml create mode 100644 templates/flavors/clusterclass-kubeadm/deleteTransformer.yaml delete mode 100644 templates/flavors/clusterclass-kubeadm/kubeadmConfigTemplate.yaml rename templates/flavors/{clusterclass-base => clusterclass-kubeadm}/linodeClusterTemplate.yaml (100%) create mode 100644 templates/flavors/clusterclass-kubeadm/replacementTransformer.yaml diff --git a/.gitignore b/.gitignore index 6b9366061..d7b236dae 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ kubeconfig* docs/book release/* templates/cluster-template*.yaml -infrastructure-linode/* +infrastructure-*-linode/* diff --git a/hack/manifests/kubeadm.yaml b/hack/manifests/kubeadm.yaml index 9a8a4d079..7dda7acd5 100644 --- a/hack/manifests/kubeadm.yaml +++ b/hack/manifests/kubeadm.yaml @@ -10,3 +10,7 @@ kind: ControlPlaneProvider metadata: name: kubeadm namespace: kubeadm-control-plane-system +spec: + manager: + featureGates: + ClusterTopology: true diff --git a/templates/common-init-files/kustomization.yaml b/templates/common-init-files/kustomization.yaml deleted file mode 100644 index b002416a5..000000000 --- a/templates/common-init-files/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - secret.yaml diff --git a/templates/common-init-files/secret.yaml b/templates/common-init-files/secret.yaml deleted file mode 100644 index fdf30bf7d..000000000 --- a/templates/common-init-files/secret.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: common-init-files - labels: - clusterctl.cluster.x-k8s.io/move: "true" -stringData: - containerd-config.toml: | - version = 2 - imports = ["/etc/containerd/conf.d/*.toml"] - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - sandbox_image = "registry.k8s.io/pause:3.9" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - k8s-modules.conf: | - overlay - br_netfilter - sysctl-k8s.conf: | - net.bridge.bridge-nf-call-iptables = 1 - net.bridge.bridge-nf-call-ip6tables = 1 - net.ipv4.ip_forward = 1 - net.ipv6.conf.all.forwarding = 1 - kubeadm-pre-init.sh: | - #!/bin/bash - set -euo pipefail - export DEBIAN_FRONTEND=noninteractive - mkdir -p -m 755 /etc/apt/keyrings - PATCH_VERSION=$${1#[v]} - VERSION=$${PATCH_VERSION%.*} - curl -fsSL "https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key" | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list - apt-get update -y - apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd - apt-mark hold kubelet kubeadm kubectl containerd - modprobe overlay - modprobe br_netfilter - sysctl --system - if [ -d "/sys/class/net/eth1" ]; then - IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1) - sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml - fi diff --git a/templates/flavors/clusterclass-base/kustomization.yaml b/templates/flavors/clusterclass-base/kustomization.yaml deleted file mode 100644 index c5119ee48..000000000 --- a/templates/flavors/clusterclass-base/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - linodeClusterTemplate.yaml - - linodeMachineControlPlaneTemplate.yaml - - linodeMachineWorkerTemplate.yaml diff --git a/templates/flavors/clusterclass-base/linodeMachineControlPlaneTemplate.yaml b/templates/flavors/clusterclass-base/linodeMachineControlPlaneTemplate.yaml deleted file mode 100644 index 89666c070..000000000 --- a/templates/flavors/clusterclass-base/linodeMachineControlPlaneTemplate.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: LinodeMachineTemplate -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 -metadata: - name: kubeadm-control-plane -spec: - template: - spec: - # image is a required field (OpenAPI schema). - image: REPLACEME - # type is a required field (OpenAPI schema). - type: REPLACEME - # region is a required field (OpenAPI schema). - region: REPLACEME diff --git a/templates/flavors/clusterclass-base/linodeMachineWorkerTemplate.yaml b/templates/flavors/clusterclass-base/linodeMachineWorkerTemplate.yaml deleted file mode 100644 index 10da62b09..000000000 --- a/templates/flavors/clusterclass-base/linodeMachineWorkerTemplate.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 -kind: LinodeMachineTemplate -metadata: - name: kubeadm-worker -spec: - template: - spec: - # image is a required field (OpenAPI schema). - image: REPLACEME - # type is a required field (OpenAPI schema). - type: REPLACEME - # region is a required field (OpenAPI schema). - region: REPLACEME diff --git a/templates/flavors/clusterclass-kubeadm/deleteTransformer.yaml b/templates/flavors/clusterclass-kubeadm/deleteTransformer.yaml new file mode 100644 index 000000000..bedd856a6 --- /dev/null +++ b/templates/flavors/clusterclass-kubeadm/deleteTransformer.yaml @@ -0,0 +1,80 @@ +--- +# Delete LinodeCluster after values are copied to LinodeClusterTemplate +apiVersion: builtin +kind: PatchTransformer +metadata: + name: LinodeCluster-patch-delete +patch: |- + $patch: delete + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeCluster + metadata: + name: ${CLUSTER_NAME} +--- +# Delete Cluster from ../base +apiVersion: builtin +kind: PatchTransformer +metadata: + name: patch-delete-cluster +patch: |- + $patch: delete + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Cluster + metadata: + name: ${CLUSTER_NAME} +--- +# Delete kubeadmControlPlane after values are copied to KubeadmControlPlaneTemplate +apiVersion: builtin +kind: PatchTransformer +metadata: + name: kubeadm-patch-delete +patch: |- + $patch: delete + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + metadata: + name: ${CLUSTER_NAME}-control-plane +--- +# delete machineDeployment from ../base +apiVersion: builtin +kind: PatchTransformer +metadata: + name: patch-delete-machineDeployment +patch: |- + $patch: delete + apiVersion: cluster.x-k8s.io/v1beta1 + kind: MachineDeployment + metadata: + name: ${CLUSTER_NAME}-md-0 +--- +# delete extra fields from kubeadmControlPlaneTemplate +apiVersion: builtin +kind: PatchTransformer +metadata: + name: patch-delete-KubeadmControlPlaneTemplate +target: + kind: KubeadmControlPlaneTemplate +patch: |- + - op: remove + path: /spec/template/spec/machineTemplate + value: + - op: remove + path: /spec/template/spec/replicas + value: + - op: remove + path: /spec/template/spec/version + value: +#patch: |- +# $patch: delete +# apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +# kind: KubeadmControlPlaneTemplate +# metadata: +# name: kubeadm-control-plane +# spec: +# machineTemplate: +# infrastructureRef: +# apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +# kind: LinodeMachineTemplate +# name: ${CLUSTER_NAME}-control-plane +# replicas: ${CONTROL_PLANE_MACHINE_COUNT} +# version: ${KUBERNETES_VERSION} \ No newline at end of file diff --git a/templates/flavors/clusterclass-kubeadm/kubeadmConfigTemplate.yaml b/templates/flavors/clusterclass-kubeadm/kubeadmConfigTemplate.yaml deleted file mode 100644 index 74621f524..000000000 --- a/templates/flavors/clusterclass-kubeadm/kubeadmConfigTemplate.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: kubeadm-worker -spec: - template: - spec: - files: - - path: /etc/containerd/config.toml - contentFrom: - secret: - name: common-init-files - key: containerd-config.toml - - path: /etc/modules-load.d/k8s.conf - contentFrom: - secret: - name: common-init-files - key: k8s-modules.conf - - path: /etc/sysctl.d/k8s.conf - contentFrom: - secret: - name: common-init-files - key: sysctl-k8s.conf - - path: /kubeadm-pre-init.sh - contentFrom: - secret: - name: common-init-files - key: kubeadm-pre-init.sh - permissions: "0500" - preKubeadmCommands: - - /kubeadm-pre-init.sh ${KUBERNETES_VERSION} - - sed -i '/swap/d' /etc/fstab - - swapoff -a - - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - name: '{{ ds.meta_data.label }}' diff --git a/templates/flavors/clusterclass-kubeadm/kustomization.yaml b/templates/flavors/clusterclass-kubeadm/kustomization.yaml index cac281973..5e157df22 100644 --- a/templates/flavors/clusterclass-kubeadm/kustomization.yaml +++ b/templates/flavors/clusterclass-kubeadm/kustomization.yaml @@ -3,9 +3,48 @@ kind: Kustomization resources: - clusterClass.yaml - kubeadmControlPlaneTemplate.yaml - - kubeadmConfigTemplate.yaml - - ../clusterclass-base - - ../../common-init-files - - ../../addons/cilium - - ../../addons/provider-linode - - ../../addons/cluster-resource-set + - linodeClusterTemplate.yaml + - ../default + +transformers: + - replacementTransformer.yaml + - deleteTransformer.yaml + +patches: + + - target: + group: infrastructure.cluster.x-k8s.io + version: v1alpha1 + kind: LinodeMachineTemplate + name: .*md-0 + options: + allowNameChange: true + patch: |- + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeMachineTemplate + metadata: + name: kubeadm-worker + + - target: + group: infrastructure.cluster.x-k8s.io + version: v1alpha1 + kind: LinodeMachineTemplate + name: .*control-plane + options: + allowNameChange: true + patch: |- + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeMachineTemplate + metadata: + name: kubeadm-control-plane + + - target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + options: + allowNameChange: true + patch: |- + kind: KubeadmConfigTemplate + metadata: + name: kubeadm-worker \ No newline at end of file diff --git a/templates/flavors/clusterclass-base/linodeClusterTemplate.yaml b/templates/flavors/clusterclass-kubeadm/linodeClusterTemplate.yaml similarity index 100% rename from templates/flavors/clusterclass-base/linodeClusterTemplate.yaml rename to templates/flavors/clusterclass-kubeadm/linodeClusterTemplate.yaml diff --git a/templates/flavors/clusterclass-kubeadm/replacementTransformer.yaml b/templates/flavors/clusterclass-kubeadm/replacementTransformer.yaml new file mode 100644 index 000000000..3b872f80c --- /dev/null +++ b/templates/flavors/clusterclass-kubeadm/replacementTransformer.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: builtin +kind: ReplacementTransformer +metadata: + name: linode-cluster-replace +replacements: + - source: + kind: LinodeCluster + fieldPath: spec + targets: + - select: + kind: LinodeClusterTemplate + fieldPaths: + - spec.template.spec +--- +apiVersion: builtin +kind: ReplacementTransformer +metadata: + name: kubeadm-controlplane-replace +replacements: + - source: + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + fieldPath: spec + targets: + - select: + kind: KubeadmControlPlaneTemplate + name: kubeadm-control-plane + fieldPaths: + - spec.template.spec \ No newline at end of file diff --git a/templates/flavors/default/kubeadmConfigTemplate.yaml b/templates/flavors/default/kubeadmConfigTemplate.yaml index 0fd5c0fbf..11852b994 100644 --- a/templates/flavors/default/kubeadmConfigTemplate.yaml +++ b/templates/flavors/default/kubeadmConfigTemplate.yaml @@ -8,25 +8,46 @@ spec: spec: files: - path: /etc/containerd/config.toml - contentFrom: - secret: - name: common-init-files - key: containerd-config.toml + content: | + version = 2 + imports = ["/etc/containerd/conf.d/*.toml"] + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.9" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true - path: /etc/modules-load.d/k8s.conf - contentFrom: - secret: - name: common-init-files - key: k8s-modules.conf + content: | + overlay + br_netfilter - path: /etc/sysctl.d/k8s.conf - contentFrom: - secret: - name: common-init-files - key: sysctl-k8s.conf + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 - path: /kubeadm-pre-init.sh - contentFrom: - secret: - name: common-init-files - key: kubeadm-pre-init.sh + content: | + #!/bin/bash + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + mkdir -p -m 755 /etc/apt/keyrings + PATCH_VERSION=$${1#[v]} + VERSION=$${PATCH_VERSION%.*} + curl -fsSL "https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key" | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd + apt-mark hold kubelet kubeadm kubectl containerd + modprobe overlay + modprobe br_netfilter + sysctl --system + if [ -d "/sys/class/net/eth1" ]; then + IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1) + sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml + fi permissions: "0500" preKubeadmCommands: - /kubeadm-pre-init.sh ${KUBERNETES_VERSION} diff --git a/templates/flavors/default/kubeadmControlPlane.yaml b/templates/flavors/default/kubeadmControlPlane.yaml index 72089e5e2..3dd930ac1 100644 --- a/templates/flavors/default/kubeadmControlPlane.yaml +++ b/templates/flavors/default/kubeadmControlPlane.yaml @@ -13,25 +13,46 @@ spec: kubeadmConfigSpec: files: - path: /etc/containerd/config.toml - contentFrom: - secret: - name: common-init-files - key: containerd-config.toml + content: | + version = 2 + imports = ["/etc/containerd/conf.d/*.toml"] + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.9" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true - path: /etc/modules-load.d/k8s.conf - contentFrom: - secret: - name: common-init-files - key: k8s-modules.conf + content: | + overlay + br_netfilter - path: /etc/sysctl.d/k8s.conf - contentFrom: - secret: - name: common-init-files - key: sysctl-k8s.conf + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 - path: /kubeadm-pre-init.sh - contentFrom: - secret: - name: common-init-files - key: kubeadm-pre-init.sh + content: | + #!/bin/bash + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + mkdir -p -m 755 /etc/apt/keyrings + PATCH_VERSION=$${1#[v]} + VERSION=$${PATCH_VERSION%.*} + curl -fsSL "https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key" | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd + apt-mark hold kubelet kubeadm kubectl containerd + modprobe overlay + modprobe br_netfilter + sysctl --system + if [ -d "/sys/class/net/eth1" ]; then + IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1) + sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml + fi permissions: "0500" preKubeadmCommands: - /kubeadm-pre-init.sh ${KUBERNETES_VERSION} diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index b0b3b36e3..2fc3dc7dd 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -4,7 +4,6 @@ resources: - ../base - kubeadmConfigTemplate.yaml - kubeadmControlPlane.yaml - - ../../common-init-files - ../../addons/cilium - ../../addons/csi-driver-linode - ../../addons/provider-linode