From 67d688567493c6d92f9c45e1902ca549b9f0d7db Mon Sep 17 00:00:00 2001 From: Ashley Dumaine Date: Tue, 30 Apr 2024 09:41:37 -0400 Subject: [PATCH] Add support for IP sharing for cilium BGP LB --- docs/src/SUMMARY.md | 1 + docs/src/topics/flavors/cilium-bgp-lb.md | 75 +++++++++++++++++++ .../cilium-bgp-lb/kubeadmConfigTemplate.yaml | 62 +++++++++++++++ .../flavors/cilium-bgp-lb/kustomization.yaml | 29 +++++++ .../cilium-bgp-lb/linodeMachineTemplate.yaml | 16 ++++ .../cilium-bgp-lb/machineDeployment.yaml | 21 ++++++ 6 files changed, 204 insertions(+) create mode 100644 docs/src/topics/flavors/cilium-bgp-lb.md create mode 100644 templates/flavors/cilium-bgp-lb/kubeadmConfigTemplate.yaml create mode 100644 templates/flavors/cilium-bgp-lb/kustomization.yaml create mode 100644 templates/flavors/cilium-bgp-lb/linodeMachineTemplate.yaml create mode 100644 templates/flavors/cilium-bgp-lb/machineDeployment.yaml diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index c3e779b38..a1c216a6a 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -11,6 +11,7 @@ - [Etcd-disk (kubeadm)](./topics/flavors/etcd-disk.md) - [ClusterClass kubeadm](./topics/flavors/clusterclass-kubeadm.md) - [Cluster Autoscaler (kubeadm)](./topics/flavors/cluster-autoscaler.md) + - [Cilium BGP LB (kubeadm)](./topics/flavors/cilium-bgp-lb.md) - [k3s](./topics/flavors/k3s.md) - [rke2](./topics/flavors/rke2.md) - [vpcless (kubeadm)](./topics/flavors/vpcless.md) diff --git a/docs/src/topics/flavors/cilium-bgp-lb.md b/docs/src/topics/flavors/cilium-bgp-lb.md new file mode 100644 index 000000000..5eff6af2d --- /dev/null +++ b/docs/src/topics/flavors/cilium-bgp-lb.md @@ -0,0 +1,75 @@ +# Cilium BGP Load-Balancing + +This flavor creates special labeled worker nodes for ingress which leverage Cilium's +[BGP Control Plane](https://docs.cilium.io/en/stable/network/bgp-control-plane/) +and [LB IPAM](https://docs.cilium.io/en/stable/network/lb-ipam/) support. + +With this flavor, Services exposed via `type: LoadBalancer` automatically get +assigned an `ExternalIP` provisioned as a shared IP through the +[linode-CCM](https://github.com/linode/linode-cloud-controller-manager/blob/shared-ip/README.md#shared-ip-load-balancing), +which is deployed with the necessary settings to perform shared IP load-balancing. + +```admonish warning +There are several important caveats to load balancing support based on current +Linode networking and API limitations: + +1. **Services with external IPv6 addresses are not reachable** + + While it is possible on a dual-stack cluster to use Cilium's LB IPAM and + BGP Control Plane features to automatically assign a IPv6 address to a + Service, this address will not be accessible outside the cluster since + Cilium will try to advertise a /128 address that gets filtered on the routing + tables for the BGP routers. +2. **Ingress traffic will not be split between BGP peer nodes** + + [Equal-Cost Multi-Path (ECMP)](https://en.wikipedia.org/wiki/Equal-cost_multi-path_routing) + is not supported on the BGP routers so ingress traffic will not be split between each + BGP Node in the cluster. One Node will be actively receiving traffic and the other(s) + will act as standby(s). +3. **Customer support is required to use this feature at this time** + + Since this uses additional IPv4 addresses on the nodes participating in Cilium's + BGPPeeringPolicy, you need to [contact our Support team](https://www.linode.com/support/) + to be permitted to add extra IPs. + +``` + +## Specification + +| Control Plane | CNI | Default OS | Installs ClusterClass | IPv4 | IPv6 | +|---------------|--------|--------------|-----------------------|------|------| +| Kubeadm | Cilium | Ubuntu 22.04 | No | Yes | No | + + +## Prerequisites + +1. [Quickstart](../getting-started.md) completed + +## Usage + +1. (Optional) Set up environment variables + ```sh + # Optional + export LINODE_BGP_PEER_MACHINE_TYPE=g6-standard-2 + export BGP_PEER_MACHINE_COUNT=2 + ``` + +2. Generate cluster yaml + + ```sh + clusterctl generate cluster test-cluster \ + --kubernetes-version v1.29.1 \ + --infrastructure linode-linode \ + --flavor cilium-bgp-lb > test-cluster.yaml + ``` + +3. Apply cluster yaml + + ```sh + kubectl apply -f test-cluster.yaml + ``` + +After the cluster exists, you can create a Service exposed with `type: LoadBalancer` and +it will automatically get assigned an ExternalIP. It's recommended to set up an ingress controller +(e.g. [https://docs.cilium.io/en/stable/network/servicemesh/ingress/](https://docs.cilium.io/en/stable/network/servicemesh/ingress/)) +to avoid needing to expose multiple `LoadBalancer` Services within the cluster. diff --git a/templates/flavors/cilium-bgp-lb/kubeadmConfigTemplate.yaml b/templates/flavors/cilium-bgp-lb/kubeadmConfigTemplate.yaml new file mode 100644 index 000000000..711008695 --- /dev/null +++ b/templates/flavors/cilium-bgp-lb/kubeadmConfigTemplate.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-bgp +spec: + template: + spec: + files: + - path: /etc/containerd/config.toml + content: | + version = 2 + imports = ["/etc/containerd/conf.d/*.toml"] + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.9" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + - path: /etc/modules-load.d/k8s.conf + content: | + overlay + br_netfilter + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 + - path: /kubeadm-pre-init.sh + content: | + #!/bin/bash + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + mkdir -p -m 755 /etc/apt/keyrings + PATCH_VERSION=$${1#[v]} + VERSION=$${PATCH_VERSION%.*} + curl -fsSL "https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/Release.key" | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$VERSION/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list + apt-get update -y + apt-get install -y kubelet=$PATCH_VERSION* kubeadm=$PATCH_VERSION* kubectl=$PATCH_VERSION* containerd + apt-mark hold kubelet kubeadm kubectl containerd + modprobe overlay + modprobe br_netfilter + sysctl --system + if [ -d "/sys/class/net/eth1" ]; then + IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1) + sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml + fi + permissions: "0500" + preKubeadmCommands: + - /kubeadm-pre-init.sh ${KUBERNETES_VERSION} + - sed -i '/swap/d' /etc/fstab + - swapoff -a + - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + node-labels: cilium-bgp-peering="true" + name: '{{ ds.meta_data.label }}' diff --git a/templates/flavors/cilium-bgp-lb/kustomization.yaml b/templates/flavors/cilium-bgp-lb/kustomization.yaml new file mode 100644 index 000000000..1d49902e2 --- /dev/null +++ b/templates/flavors/cilium-bgp-lb/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../default + - machineDeployment.yaml + - linodeMachineTemplate.yaml + - kubeadmConfigTemplate.yaml +patches: + - target: + kind: HelmChartProxy + name: .*-linode-cloud-controller-manager + patch: |- + - op: replace + path: /spec/valuesTemplate + value: | + sharedIPLoadBalancing: + defaultLoadBalancer: cilium-bgp + bgpNodeSelector: cilium-bgp-peering=true + routeController: + vpcName: {{ .InfraCluster.spec.vpcRef.name }} + clusterCIDR: 10.0.0.0/8 + configureCloudRoutes: true + secretRef: + name: "linode-token-region" + image: + pullPolicy: IfNotPresent + env: + - name: LINODE_URL + value: https://api.linode.com/v4beta diff --git a/templates/flavors/cilium-bgp-lb/linodeMachineTemplate.yaml b/templates/flavors/cilium-bgp-lb/linodeMachineTemplate.yaml new file mode 100644 index 000000000..1e8b5e49f --- /dev/null +++ b/templates/flavors/cilium-bgp-lb/linodeMachineTemplate.yaml @@ -0,0 +1,16 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-bgp +spec: + template: + spec: + image: ${LINODE_OS:="linode/ubuntu22.04"} + type: ${LINODE_BGP_PEER_MACHINE_TYPE:=${LINODE_MACHINE_TYPE}} + region: ${LINODE_REGION} + interfaces: + - purpose: public + primary: true + authorizedKeys: + # uncomment to include your ssh key in linode provisioning + # - ${LINODE_SSH_PUBKEY:=""} diff --git a/templates/flavors/cilium-bgp-lb/machineDeployment.yaml b/templates/flavors/cilium-bgp-lb/machineDeployment.yaml new file mode 100644 index 000000000..b1993e95f --- /dev/null +++ b/templates/flavors/cilium-bgp-lb/machineDeployment.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-bgp +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${BGP_PEER_MACHINE_COUNT:=2} + template: + spec: + clusterName: ${CLUSTER_NAME} + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: ${CLUSTER_NAME}-md-bgp + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: ${CLUSTER_NAME}-md-bgp + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeMachineTemplate