From 5cb4aef074aba91b6df6651a1601d4c6fc9518a7 Mon Sep 17 00:00:00 2001 From: Ashley Dumaine <5779804+AshleyDumaine@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:20:18 -0500 Subject: [PATCH] [feat]: add cluster template, use CAAPH to install cilium for the CNI (#96) * add cluster template with bootstrap scripts, use CAAPH to install cilium * use swapoff instead of trying to set swap disk to 0 to speed up linode booting --- Tiltfile | 2 +- .../linodemachine_controller_helpers.go | 1 - templates/addons/cilium-helm.yaml | 22 ++ templates/cluster-template.yaml | 207 ++++++++++++++++++ 4 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 templates/addons/cilium-helm.yaml create mode 100644 templates/cluster-template.yaml diff --git a/Tiltfile b/Tiltfile index d67d27d0a..ee0db0ccd 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,7 +2,7 @@ docker_build("controller", ".", only=("Dockerfile", "Makefile", "vendor","go.mod local_resource( 'capi-controller-manager', - cmd='clusterctl init', + cmd='clusterctl init --addon helm', ) k8s_yaml(kustomize('config/default')) diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index 770eafe04..0bd8d8579 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -57,7 +57,6 @@ func (*LinodeMachineReconciler) newCreateConfig(ctx context.Context, machineScop return nil, err } - createConfig.SwapSize = util.Pointer(0) createConfig.PrivateIP = true bootstrapData, err := machineScope.GetBootstrapData(ctx) diff --git a/templates/addons/cilium-helm.yaml b/templates/addons/cilium-helm.yaml new file mode 100644 index 000000000..da0f2bfa1 --- /dev/null +++ b/templates/addons/cilium-helm.yaml @@ -0,0 +1,22 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cilium +spec: + clusterSelector: + matchLabels: + cni: cilium + repoURL: https://helm.cilium.io/ + chartName: cilium + version: 1.15.0 + options: + waitForJobs: true + wait: true + timeout: 5m + valuesTemplate: | + hubble: + relay: + enabled: true + ui: + enabled: true +--- diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml new file mode 100644 index 000000000..075d75208 --- /dev/null +++ b/templates/cluster-template.yaml @@ -0,0 +1,207 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + cni: cilium +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.128.0/17 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeCluster +metadata: + name: ${CLUSTER_NAME} +spec: + region: ${LINODE_REGION} +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: LinodeMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + files: + - path: /etc/containerd/config.toml + contentFrom: + secret: + name: common-init-files + key: containerd-config.toml + - path: /etc/modules-load.d/k8s.conf + contentFrom: + secret: + name: common-init-files + key: k8s-modules.conf + - path: /etc/sysctl.d/k8s.conf + contentFrom: + secret: + name: common-init-files + key: sysctl-k8s.conf + - path: /kubeadm-pre-init.sh + contentFrom: + secret: + name: common-init-files + key: kubeadm-pre-init.sh + permissions: "0500" + preKubeadmCommands: + - /kubeadm-pre-init.sh '{{ ds.meta_data.label }}' "${KUBERNETES_VERSION}" + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: 'linode:///{{ ds.meta_data.region }}/{{ ds.meta_data.id }}' + name: '{{ ds.meta_data.label }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: 'linode:///{{ ds.meta_data.region }}/{{ ds.meta_data.id }}' + name: '{{ ds.meta_data.label }}' + version: "${KUBERNETES_VERSION}" +--- +kind: LinodeMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + image: ${LINODE_OS} + type: ${LINODE_CONTROL_PLANE_MACHINE_TYPE} + region: ${LINODE_REGION} + authorizedKeys: + - ${LINODE_SSH_KEY} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: ${CLUSTER_NAME} + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: ${CLUSTER_NAME}-md-0 + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: ${CLUSTER_NAME}-md-0 + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + image: ${LINODE_OS} + type: ${LINODE_MACHINE_TYPE} + region: ${LINODE_REGION} + authorizedKeys: + - ${LINODE_SSH_KEY} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + files: + - path: /etc/containerd/config.toml + contentFrom: + secret: + name: common-init-files + key: containerd-config.toml + - path: /etc/modules-load.d/k8s.conf + contentFrom: + secret: + name: common-init-files + key: k8s-modules.conf + - path: /etc/sysctl.d/k8s.conf + contentFrom: + secret: + name: common-init-files + key: sysctl-k8s.conf + - path: /kubeadm-pre-init.sh + contentFrom: + secret: + name: common-init-files + key: kubeadm-pre-init.sh + permissions: "0500" + preKubeadmCommands: + - /kubeadm-pre-init.sh '{{ ds.meta_data.label }}' "${KUBERNETES_VERSION}" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: 'linode:///{{ ds.meta_data.region }}/{{ ds.meta_data.id }}' + name: '{{ ds.meta_data.label }}' +--- +apiVersion: v1 +kind: Secret +metadata: + name: common-init-files +stringData: + containerd-config.toml: | + version = 2 + imports = ["/etc/containerd/conf.d/*.toml"] + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.9" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + k8s-modules.conf: | + overlay + br_netfilter + sysctl-k8s.conf: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + kubeadm-pre-init.sh: | + #!/bin/bash + export DEBIAN_FRONTEND=noninteractive + hostnamectl set-hostname "$1" && hostname -F /etc/hostname + mkdir -p -m 755 /etc/apt/keyrings + VERSION=${2%.*} + curl -fsSL "https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key" | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + apt-get install -y kubelet=$2* kubeadm=$2* kubectl=$2* containerd + apt-mark hold kubelet kubeadm kubectl containerd + modprobe overlay + modprobe br_netfilter + sysctl --system + sed -i '/swap/d' /etc/fstab + swapoff -a