From b9b0395b9c4dc58965afec267f1a271ccba74305 Mon Sep 17 00:00:00 2001 From: Ashley Dumaine <5779804+AshleyDumaine@users.noreply.github.com> Date: Fri, 7 Jun 2024 09:51:10 -0400 Subject: [PATCH] [feat] [deps]: New flavor for Cilium BGP load-balancing for Services, bump linode-CCM version (#317) * Add support for IP sharing for cilium BGP LB * update helm chart option * use kustomize transformers * update for new flavor generation * support setting node-ip for vpcless * formatting, need dual-stack and vpcless for bgp sessions to go to established * bump version, update docs * bump ccm version * remove conditional * inherit from dual-stack flavor for cilium-bgp-lb * update docs * update GHA filters --- .github/filters.yml | 2 + docs/src/SUMMARY.md | 1 + docs/src/topics/flavors/cilium-bgp-lb.md | 76 +++++++++++++++ templates/addons/ccm-linode/ccm-linode.yaml | 2 +- .../cilium-bgp-lb/kubeadmConfigTemplate.yaml | 5 + .../kubeadm/cilium-bgp-lb/kustomization.yaml | 96 +++++++++++++++++++ .../cilium-bgp-lb/linodeMachineTemplate.yaml | 4 + .../cilium-bgp-lb/machineDeployment.yaml | 8 ++ 8 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 docs/src/topics/flavors/cilium-bgp-lb.md create mode 100644 templates/flavors/kubeadm/cilium-bgp-lb/kubeadmConfigTemplate.yaml create mode 100644 templates/flavors/kubeadm/cilium-bgp-lb/kustomization.yaml create mode 100644 templates/flavors/kubeadm/cilium-bgp-lb/linodeMachineTemplate.yaml create mode 100644 templates/flavors/kubeadm/cilium-bgp-lb/machineDeployment.yaml diff --git a/.github/filters.yml b/.github/filters.yml index 326b5dede..1303617ab 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -20,6 +20,8 @@ kubeadm_self-healing: - templates/flavors/kubeadm/self-healing/* kubeadm_vpcless: - templates/flavors/kubeadm/vpcless/* +kubeadm_cilium_bgp_lb: + - templates/flavors/kubeadm/cilium-bgp-lb/* k3s: - templates/flavors/k3s/default/* diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 790aa9d15..065964df9 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -11,6 +11,7 @@ - [Etcd-disk](./topics/flavors/etcd-disk.md) - [ClusterClass kubeadm](./topics/flavors/clusterclass-kubeadm.md) - [Cluster Autoscaler](./topics/flavors/cluster-autoscaler.md) + - [Cilium BGP LB (kubeadm)](./topics/flavors/cilium-bgp-lb.md) - [k3s](./topics/flavors/k3s.md) - [rke2](./topics/flavors/rke2.md) - [vpcless](./topics/flavors/vpcless.md) diff --git a/docs/src/topics/flavors/cilium-bgp-lb.md b/docs/src/topics/flavors/cilium-bgp-lb.md new file mode 100644 index 000000000..9dd891c3c --- /dev/null +++ b/docs/src/topics/flavors/cilium-bgp-lb.md @@ -0,0 +1,76 @@ +# Cilium BGP Load-Balancing + +This flavor creates special labeled worker nodes for ingress which leverage Cilium's +[BGP Control Plane](https://docs.cilium.io/en/stable/network/bgp-control-plane/) +and [LB IPAM](https://docs.cilium.io/en/stable/network/lb-ipam/) support. + +With this flavor, Services exposed via `type: LoadBalancer` automatically get +assigned an `ExternalIP` provisioned as a shared IP through the +[linode-CCM](https://github.com/linode/linode-cloud-controller-manager?tab=readme-ov-file#shared-ip-load-balancing), +which is deployed with the necessary settings to perform shared IP load-balancing. + +```admonish warning +There are a couple important caveats to load balancing support based on current +Linode networking and API limitations: + +1. **Ingress traffic will not be split between BGP peer nodes** + + [Equal-Cost Multi-Path (ECMP)](https://en.wikipedia.org/wiki/Equal-cost_multi-path_routing) + is not supported on the BGP routers so ingress traffic will not be split between each + BGP Node in the cluster. One Node will be actively receiving traffic and the other(s) + will act as standby(s). +2. **Customer support is required to use this feature at this time** + + Since this uses additional IPv4 addresses on the nodes participating in Cilium's + BGPPeeringPolicy, you need to [contact our Support team](https://www.linode.com/support/) + to be permitted to add extra IPs. + +``` + +```admonish note +Dual-stack support is enabled for clusters using this flavor since IPv6 is used for router +and neighbor solicitation. + +Without enabling dual-stack support, the IPv6 traffic is blocked if the Cilium host firewall +is enabled (which it is by default in CAPL), even if there are no configured `CiliumClusterWideNetworkPolicies` +or the policy is set to audit (default) instead of enforce (see [https://github.com/cilium/cilium/issues/27484](https://github.com/cilium/cilium/issues/27484)). More information about firewalling can be found on the [Firewalling](../firewalling.md) page. +``` + +## Specification + +| Control Plane | CNI | Default OS | Installs ClusterClass | IPv4 | IPv6 | +|---------------|--------|--------------|-----------------------|------|------| +| Kubeadm | Cilium | Ubuntu 22.04 | No | Yes | Yes | + + +## Prerequisites + +1. [Quickstart](../getting-started.md) completed + +## Usage + +1. (Optional) Set up environment variable + ```sh + # Optional + export BGP_PEER_MACHINE_COUNT=2 + ``` + +2. Generate cluster yaml + + ```sh + clusterctl generate cluster test-cluster \ + --kubernetes-version v1.29.1 \ + --infrastructure linode-linode \ + --flavor kubeadm-cilium-bgp-lb > test-cluster.yaml + ``` + +3. Apply cluster yaml + + ```sh + kubectl apply -f test-cluster.yaml + ``` + +After the cluster exists, you can create a Service exposed with `type: LoadBalancer` and +it will automatically get assigned an ExternalIP. It's recommended to set up an ingress controller +(e.g. [https://docs.cilium.io/en/stable/network/servicemesh/ingress/](https://docs.cilium.io/en/stable/network/servicemesh/ingress/)) +to avoid needing to expose multiple `LoadBalancer` Services within the cluster. diff --git a/templates/addons/ccm-linode/ccm-linode.yaml b/templates/addons/ccm-linode/ccm-linode.yaml index 1f2e23642..cc7f9cb5e 100644 --- a/templates/addons/ccm-linode/ccm-linode.yaml +++ b/templates/addons/ccm-linode/ccm-linode.yaml @@ -9,7 +9,7 @@ spec: repoURL: https://linode.github.io/linode-cloud-controller-manager/ chartName: ccm-linode namespace: kube-system - version: ${LINODE_CCM_VERSION:=v0.4.4} + version: ${LINODE_CCM_VERSION:=v0.4.6} options: waitForJobs: true wait: true diff --git a/templates/flavors/kubeadm/cilium-bgp-lb/kubeadmConfigTemplate.yaml b/templates/flavors/kubeadm/cilium-bgp-lb/kubeadmConfigTemplate.yaml new file mode 100644 index 000000000..2159ad300 --- /dev/null +++ b/templates/flavors/kubeadm/cilium-bgp-lb/kubeadmConfigTemplate.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-bgp diff --git a/templates/flavors/kubeadm/cilium-bgp-lb/kustomization.yaml b/templates/flavors/kubeadm/cilium-bgp-lb/kustomization.yaml new file mode 100644 index 000000000..7e0f7010f --- /dev/null +++ b/templates/flavors/kubeadm/cilium-bgp-lb/kustomization.yaml @@ -0,0 +1,96 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../dual-stack + - machineDeployment.yaml + - linodeMachineTemplate.yaml + - kubeadmConfigTemplate.yaml + +patches: + - target: + kind: HelmChartProxy + name: .*-linode-cloud-controller-manager + patch: |- + - op: replace + path: /spec/valuesTemplate + value: | + sharedIPLoadBalancing: + loadBalancerType: cilium-bgp + bgpNodeSelector: cilium-bgp-peering=true + secretRef: + name: "linode-token-region" + image: + pullPolicy: IfNotPresent + env: + - name: LINODE_URL + value: https://api.linode.com/v4beta +transformers: + - | + apiVersion: builtin + kind: ReplacementTransformer + metadata: + name: clone-bgp + replacements: + - source: + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + fieldPath: .spec + targets: + - select: + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-bgp + fieldPaths: + - .spec + options: + create: true + - source: + kind: LinodeMachineTemplate + name: ${CLUSTER_NAME}-md-0 + fieldPath: .spec + targets: + - select: + kind: LinodeMachineTemplate + name: ${CLUSTER_NAME}-md-bgp + fieldPaths: + - .spec + options: + create: true + - source: + kind: MachineDeployment + name: ${CLUSTER_NAME}-md-0 + fieldPath: .spec.template + targets: + - select: + kind: MachineDeployment + name: ${CLUSTER_NAME}-md-bgp + fieldPaths: + - .spec.template + options: + create: true + - |- + apiVersion: builtin + kind: PatchTransformer + metadata: + name: patch-kubeadmconfig + patch: |- + - op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/node-labels + value: cilium-bgp-peering="true" + target: + kind: KubeadmConfigTemplate + name: .*-md-bgp + - |- + apiVersion: builtin + kind: PatchTransformer + metadata: + name: patch-machinedeployment + patch: |- + - op: replace + path: /spec/template/spec/bootstrap/configRef/name + value: ${CLUSTER_NAME}-md-bgp + - op: replace + path: /spec/template/spec/infrastructureRef/name + value: ${CLUSTER_NAME}-md-bgp + target: + kind: MachineDeployment + name: .*-md-bgp diff --git a/templates/flavors/kubeadm/cilium-bgp-lb/linodeMachineTemplate.yaml b/templates/flavors/kubeadm/cilium-bgp-lb/linodeMachineTemplate.yaml new file mode 100644 index 000000000..943988db0 --- /dev/null +++ b/templates/flavors/kubeadm/cilium-bgp-lb/linodeMachineTemplate.yaml @@ -0,0 +1,4 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-bgp diff --git a/templates/flavors/kubeadm/cilium-bgp-lb/machineDeployment.yaml b/templates/flavors/kubeadm/cilium-bgp-lb/machineDeployment.yaml new file mode 100644 index 000000000..c63ba0d89 --- /dev/null +++ b/templates/flavors/kubeadm/cilium-bgp-lb/machineDeployment.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-bgp +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${BGP_PEER_MACHINE_COUNT:=2}