From 5ec454f50e7a45113ddae05e68728ec622b0331f Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 15 Oct 2024 22:35:33 -0700 Subject: [PATCH] [Release-1.30] Revert "Make svclb as simple as possible" (#11113) * Revert "Make svclb as simple as possible" This reverts commit 1befd65a0a0ef387546fd1b9ca53b3dec4823f46. Signed-off-by: manuelbuil Signed-off-by: Derek Nola * Pin E2E tests to 22.04 Signed-off-by: Derek Nola --------- Signed-off-by: manuelbuil Signed-off-by: Derek Nola Co-authored-by: manuelbuil --- .github/workflows/e2e.yaml | 4 +- .github/workflows/unitcoverage.yaml | 2 +- pkg/cloudprovider/servicelb.go | 93 ++++++++++++++++++++++----- scripts/airgap/image-list.txt | 1 + updatecli/updatecli.d/klipper-lb.yaml | 71 ++++++++++++++++++++ updatecli/values.yaml | 4 ++ 6 files changed, 156 insertions(+), 19 deletions(-) create mode 100644 updatecli/updatecli.d/klipper-lb.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ae06960907d4..56b1328a1e01 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -33,12 +33,12 @@ jobs: e2e: name: "E2E Tests" needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 40 strategy: fail-fast: false matrix: - etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm, svcpoliciesandfirewall] + etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm] max-parallel: 3 steps: - name: "Checkout" diff --git a/.github/workflows/unitcoverage.yaml b/.github/workflows/unitcoverage.yaml index 7f30a6ef57dd..5bbc00f2acce 100644 --- a/.github/workflows/unitcoverage.yaml +++ b/.github/workflows/unitcoverage.yaml @@ -28,7 +28,7 @@ permissions: jobs: test: name: Unit Tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 20 steps: - name: Checkout diff --git a/pkg/cloudprovider/servicelb.go b/pkg/cloudprovider/servicelb.go index 3aa0fa95ab70..0f2e6d4bae97 100644 --- a/pkg/cloudprovider/servicelb.go +++ b/pkg/cloudprovider/servicelb.go @@ -2,12 +2,12 @@ package cloudprovider import ( "context" - "encoding/json" "fmt" "sort" + "strconv" "strings" "time" - + "encoding/json" "sigs.k8s.io/yaml" "github.com/k3s-io/k3s/pkg/util" @@ -43,7 +43,6 @@ var ( daemonsetNodeLabel = "svccontroller." + version.Program + ".cattle.io/enablelb" daemonsetNodePoolLabel = "svccontroller." + version.Program + ".cattle.io/lbpool" nodeSelectorLabel = "svccontroller." + version.Program + ".cattle.io/nodeselector" - extTrafficPolicyLabel = "svccontroller." + version.Program + ".cattle.io/exttrafficpolicy" priorityAnnotation = "svccontroller." + version.Program + ".cattle.io/priorityclassname" tolerationsAnnotation = "svccontroller." + version.Program + ".cattle.io/tolerations" controllerName = names.ServiceLBController @@ -56,7 +55,7 @@ const ( ) var ( - DefaultLBImage = "rancher/mirrored-library-busybox:1.36.1" + DefaultLBImage = "rancher/klipper-lb:v0.4.9" ) func (k *k3s) Register(ctx context.Context, @@ -436,17 +435,35 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { oneInt := intstr.FromInt(1) priorityClassName := k.getPriorityClassName(svc) localTraffic := servicehelper.RequestsOnlyLocalTraffic(svc) + sourceRangesSet, err := servicehelper.GetLoadBalancerSourceRanges(svc) + if err != nil { + return nil, err + } + sourceRanges := strings.Join(sourceRangesSet.StringSlice(), ",") securityContext := &core.PodSecurityContext{} + for _, ipFamily := range svc.Spec.IPFamilies { + switch ipFamily { + case core.IPv4Protocol: + securityContext.Sysctls = append(securityContext.Sysctls, core.Sysctl{Name: "net.ipv4.ip_forward", Value: "1"}) + case core.IPv6Protocol: + securityContext.Sysctls = append(securityContext.Sysctls, core.Sysctl{Name: "net.ipv6.conf.all.forwarding", Value: "1"}) + if sourceRanges == "0.0.0.0/0" { + // The upstream default load-balancer source range only includes IPv4, even if the service is IPv6-only or dual-stack. + // If using the default range, and IPv6 is enabled, also allow IPv6. + sourceRanges += ",::/0" + } + } + } + ds := &apps.DaemonSet{ ObjectMeta: meta.ObjectMeta{ Name: name, Namespace: k.LBNamespace, Labels: labels.Set{ - nodeSelectorLabel: "false", - svcNameLabel: svc.Name, - svcNamespaceLabel: svc.Namespace, - extTrafficPolicyLabel: "Cluster", + nodeSelectorLabel: "false", + svcNameLabel: svc.Name, + svcNamespaceLabel: svc.Namespace, }, }, TypeMeta: meta.TypeMeta{ @@ -505,7 +522,6 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { Name: portName, Image: k.LBImage, ImagePullPolicy: core.PullIfNotPresent, - Command: []string{"sleep", "inf"}, Ports: []core.ContainerPort{ { Name: portName, @@ -514,7 +530,57 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { Protocol: port.Protocol, }, }, + Env: []core.EnvVar{ + { + Name: "SRC_PORT", + Value: strconv.Itoa(int(port.Port)), + }, + { + Name: "SRC_RANGES", + Value: sourceRanges, + }, + { + Name: "DEST_PROTO", + Value: string(port.Protocol), + }, + }, + SecurityContext: &core.SecurityContext{ + Capabilities: &core.Capabilities{ + Add: []core.Capability{ + "NET_ADMIN", + }, + }, + }, + } + + if localTraffic { + container.Env = append(container.Env, + core.EnvVar{ + Name: "DEST_PORT", + Value: strconv.Itoa(int(port.NodePort)), + }, + core.EnvVar{ + Name: "DEST_IPS", + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ + FieldPath: getHostIPsFieldPath(), + }, + }, + }, + ) + } else { + container.Env = append(container.Env, + core.EnvVar{ + Name: "DEST_PORT", + Value: strconv.Itoa(int(port.Port)), + }, + core.EnvVar{ + Name: "DEST_IPS", + Value: strings.Join(svc.Spec.ClusterIPs, ","), + }, + ) } + ds.Spec.Template.Spec.Containers = append(ds.Spec.Template.Spec.Containers, container) } @@ -542,11 +608,6 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { } ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations, tolerations...) - // Change the label to force the DaemonSet to update and call onPodChange if the ExternalTrafficPolicy changes - if localTraffic { - ds.Spec.Template.Labels[extTrafficPolicyLabel] = "Local" - } - return ds, nil } @@ -649,8 +710,8 @@ func (k *k3s) getPriorityClassName(svc *core.Service) string { return k.LBDefaultPriorityClassName } -// getTolerations retrieves the tolerations from a service's annotations. -// It parses the tolerations from a JSON or YAML string stored in the annotations. +// getTolerations retrieves the tolerations from a service's annotations. +// It parses the tolerations from a JSON or YAML string stored in the annotations. func (k *k3s) getTolerations(svc *core.Service) ([]core.Toleration, error) { tolerationsStr, ok := svc.Annotations[tolerationsAnnotation] if !ok { diff --git a/scripts/airgap/image-list.txt b/scripts/airgap/image-list.txt index 62ee4ef7adc0..3f700553e029 100644 --- a/scripts/airgap/image-list.txt +++ b/scripts/airgap/image-list.txt @@ -1,4 +1,5 @@ docker.io/rancher/klipper-helm:v0.9.3-build20241008 +docker.io/rancher/klipper-lb:v0.4.9 docker.io/rancher/local-path-provisioner:v0.0.30 docker.io/rancher/mirrored-coredns-coredns:1.11.3 docker.io/rancher/mirrored-library-busybox:1.36.1 diff --git a/updatecli/updatecli.d/klipper-lb.yaml b/updatecli/updatecli.d/klipper-lb.yaml new file mode 100644 index 000000000000..28fc57e77de2 --- /dev/null +++ b/updatecli/updatecli.d/klipper-lb.yaml @@ -0,0 +1,71 @@ +--- +name: "Bump Klipper LB version" +scms: + k3s: + kind: "github" + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + username: "{{ .github.username }}" + token: "{{ requiredEnv .github.token }}" + owner: "{{ .k3s.org }}" + repository: "{{ .k3s.repo }}" + branch: "{{ .k3s.branch }}" + commitmessage: + title: "Bump Klipper LB version" + klipper-lb: + kind: "github" + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + username: "{{ .github.username }}" + token: "{{ requiredEnv .github.token }}" + owner: "{{ .k3s.org }}" + repository: "{{ .klipper_lb.repo }}" + branch: "{{ .klipper_lb.branch }}" + +actions: + github: + title: "Bump Klipper LB version" + kind: "github/pullrequest" + scmid: "k3s" + spec: + automerge: false + mergemethod: "squash" + usetitleforautomerge: true + parent: false + labels: + - "dependencies" + +sources: + klipper-lb: + name: "Get Klipper LB latest release version" + kind: "githubrelease" + spec: + owner: "{{ .klipper_lb.org }}" + repository: "{{ .klipper_lb.repo }}" + branch: "{{ .klipper_lb.branch }}" + token: "{{ requiredEnv .github.token }}" + versionfilter: + kind: "latest" + +conditions: + klipper-lb: + name: "Check rancher/klipper-lb image version in DockerHub" + kind: "dockerimage" + sourceid: "klipper-lb" + spec: + image: "rancher/klipper-lb" + +targets: + klipper-lb: + name: "Update rancher/klipper-lb image versions" + kind: "file" + scmid: "k3s" + sourceid: "klipper-lb" + spec: + files: + - "pkg/cloudprovider/servicelb.go" + - "scripts/airgap/image-list.txt" + matchpattern: 'rancher/klipper-lb:v\d+\.\d+\.\d+(-\w+)?' + replacepattern: 'rancher/klipper-lb:{{ source "klipper-lb" }}' diff --git a/updatecli/values.yaml b/updatecli/values.yaml index 3890caedbe3e..5b46fade560b 100644 --- a/updatecli/values.yaml +++ b/updatecli/values.yaml @@ -11,6 +11,10 @@ klipper_helm: org: "k3s-io" repo: "klipper-helm" branch: "master" +klipper_lb: + org: "k3s-io" + repo: "klipper-lb" + branch: "master" local_path_provisioner: org: "rancher" repo: "local-path-provisioner"