From ec7e45422b0fdbb467af61b9868e5470915e6ae0 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Thu, 26 Sep 2024 16:12:16 +0200 Subject: [PATCH] Intoduce endpointslice controller Signed-off-by: Andrei Kvapil --- .../kubevirteps.go | 53 +++++++++++-------- cmd/kubevirt-cloud-controller-manager/main.go | 3 +- .../kubevirteps/kubevirteps_controller.go | 13 ++--- .../kubevirteps_controller_test.go | 10 ++++ pkg/provider/loadbalancer.go | 8 +-- 5 files changed, 55 insertions(+), 32 deletions(-) diff --git a/cmd/kubevirt-cloud-controller-manager/kubevirteps.go b/cmd/kubevirt-cloud-controller-manager/kubevirteps.go index e6b41fbb0..f8a5999b9 100644 --- a/cmd/kubevirt-cloud-controller-manager/kubevirteps.go +++ b/cmd/kubevirt-cloud-controller-manager/kubevirteps.go @@ -3,8 +3,10 @@ package main import ( "context" "fmt" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" @@ -37,7 +39,7 @@ func startKubevirtCloudController( return nil, false, err } - if !*kubevirtCloud.GetCloudConfig().LoadBalancer.EnableEPSController { + if kubevirtCloud.GetCloudConfig().LoadBalancer.EnableEPSController == nil || !*kubevirtCloud.GetCloudConfig().LoadBalancer.EnableEPSController { klog.Infof(fmt.Sprintf("%s is not enabled.", kubevirteps.ControllerName)) return nil, false, nil } @@ -54,37 +56,46 @@ func startKubevirtCloudController( klog.Infof("Setting up infra client.") // This is the kubeconfig for the infra cluster - infraKubeConfig, err := kubevirtCloud.GetInfraKubeconfig() - if err != nil { - klog.Errorf("Failed to get infra kubeconfig: %v", err) - return nil, false, err - } - - var infraClientConfig clientcmd.ClientConfig - infraClientConfig, err = clientcmd.NewClientConfigFromBytes([]byte(infraKubeConfig)) - if err != nil { - klog.Errorf("Failed to create client config for infra cluster: %v", err) - return nil, false, err - } - - infraConfig, err := infraClientConfig.ClientConfig() - if err != nil { - klog.Errorf("Failed to create client config for infra cluster: %v", err) - return nil, false, err + var restConfig *rest.Config + + if kubevirtCloud.GetCloudConfig().Kubeconfig == "" { + restConfig, err = rest.InClusterConfig() + if err != nil { + klog.Errorf("Failed to get in-cluster config: %v", err) + return nil, false, err + } + } else { + var infraKubeConfig string + infraKubeConfig, err = kubevirtCloud.GetInfraKubeconfig() + if err != nil { + klog.Errorf("Failed to get infra kubeconfig: %v", err) + return nil, false, err + } + var clientConfig clientcmd.ClientConfig + clientConfig, err = clientcmd.NewClientConfigFromBytes([]byte(infraKubeConfig)) + if err != nil { + klog.Errorf("Failed to create client config from infra kubeconfig: %v", err) + return nil, false, err + } + restConfig, err = clientConfig.ClientConfig() + if err != nil { + klog.Errorf("Failed to create rest config for infra cluster: %v", err) + return nil, false, err + } } var infraClient kubernetes.Interface // create new client for the infra cluster - infraClient, err = kubernetes.NewForConfig(infraConfig) + infraClient, err = kubernetes.NewForConfig(restConfig) if err != nil { - klog.Errorf("Failed to create client for infra cluster: %v", err) + klog.Errorf("Failed to create infra cluster client: %v", err) return nil, false, err } var infraDynamic dynamic.Interface - infraDynamic, err = dynamic.NewForConfig(infraConfig) + infraDynamic, err = dynamic.NewForConfig(restConfig) if err != nil { klog.Errorf("Failed to create dynamic client for infra cluster: %v", err) return nil, false, err diff --git a/cmd/kubevirt-cloud-controller-manager/main.go b/cmd/kubevirt-cloud-controller-manager/main.go index 564b52007..3f91c4cd0 100644 --- a/cmd/kubevirt-cloud-controller-manager/main.go +++ b/cmd/kubevirt-cloud-controller-manager/main.go @@ -45,14 +45,13 @@ func main() { fss := cliflag.NamedFlagSets{} controllerInitializers := app.DefaultInitFuncConstructors - controllerAliases := map[string]string{"kubevirt-eps": kubevirteps.ControllerName.String()} // add kubevirt-cloud-controller to the list of controllers controllerInitializers[kubevirteps.ControllerName.String()] = app.ControllerInitFuncConstructor{ Constructor: StartKubevirtCloudControllerWrapper, } - command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, controllerAliases, fss, wait.NeverStop) + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop) code := cli.Run(command) os.Exit(code) } diff --git a/pkg/controller/kubevirteps/kubevirteps_controller.go b/pkg/controller/kubevirteps/kubevirteps_controller.go index 9366d7c26..4dabf238b 100644 --- a/pkg/controller/kubevirteps/kubevirteps_controller.go +++ b/pkg/controller/kubevirteps/kubevirteps_controller.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "strings" + "time" + v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -25,8 +28,6 @@ import ( "k8s.io/klog/v2" kubevirtv1 "kubevirt.io/api/core/v1" kubevirt "kubevirt.io/cloud-provider-kubevirt/pkg/provider" - "strings" - "time" ) const ( @@ -312,7 +313,7 @@ func (c *Controller) getInfraEPSFromInfraService(ctx context.Context, svc *v1.Se func (c *Controller) reconcile(ctx context.Context, r *Request) error { service, ok := r.Obj.(*v1.Service) - if !ok { + if !ok || service == nil { return errors.New("could not cast object to service") } @@ -350,8 +351,8 @@ func (c *Controller) reconcile(ctx context.Context, r *Request) error { // If the services switched to a different address type, we need to delete the old ones, because it's immutable. // If the services switched to a different externalTrafficPolicy, we need to delete the old ones. for _, eps := range infraExistingEpSlices { - if service.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeLocal || serviceDeleted { - klog.Infof("Added for deletion EndpointSlice %s in namespace %s because it has an externalTrafficPolicy different from Local", eps.Name, eps.Namespace) + if service.Spec.Selector != nil || serviceDeleted { + klog.Infof("Added for deletion EndpointSlice %s in namespace %s because it has a selector", eps.Name, eps.Namespace) // to be sure we don't delete any slice that is not managed by us if c.managedByController(eps) { slicesToDelete = append(slicesToDelete, eps) @@ -595,7 +596,7 @@ func (c *Controller) newSlice(service *v1.Service, desiredPorts []discovery.Endp func (c *Controller) getDesiredEndpoints(service *v1.Service, tenantSlices []*discovery.EndpointSlice) []*discovery.Endpoint { var desiredEndpoints []*discovery.Endpoint - if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal { + if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal || service.Spec.Selector == nil { // Extract the desired endpoints from the tenant EndpointSlices // for extracting the nodes it does not matter what type of address we are dealing with // all nodes with an endpoint for a corresponding slice will be selected. diff --git a/pkg/controller/kubevirteps/kubevirteps_controller_test.go b/pkg/controller/kubevirteps/kubevirteps_controller_test.go index 7e645a0bc..23b38d369 100644 --- a/pkg/controller/kubevirteps/kubevirteps_controller_test.go +++ b/pkg/controller/kubevirteps/kubevirteps_controller_test.go @@ -2,6 +2,7 @@ package kubevirteps import ( "context" + g "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" @@ -34,6 +35,14 @@ type testKubevirtEPSController struct { } func createInfraServiceLB(name, tenantServiceName, clusterName string, servicePort v1.ServicePort, externalTrafficPolicy v1.ServiceExternalTrafficPolicy) *v1.Service { + var selector map[string]string + if externalTrafficPolicy == v1.ServiceExternalTrafficPolicyCluster { + selector = map[string]string{ + "cluster.x-k8s.io/role": "worker", + "cluster.x-k8s.io/cluster-name": clusterName, + } + } + return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -50,6 +59,7 @@ func createInfraServiceLB(name, tenantServiceName, clusterName string, servicePo }, Type: v1.ServiceTypeLoadBalancer, ExternalTrafficPolicy: externalTrafficPolicy, + Selector: selector, IPFamilies: []v1.IPFamily{ v1.IPv4Protocol, }, diff --git a/pkg/provider/loadbalancer.go b/pkg/provider/loadbalancer.go index 6048ef749..4ae521a4a 100644 --- a/pkg/provider/loadbalancer.go +++ b/pkg/provider/loadbalancer.go @@ -208,9 +208,11 @@ func (lb *loadbalancer) createLoadBalancerService(ctx context.Context, lbName st }, } // Give controller privilege above selectorless - if lb.config.EnableEPSController != nil && *lb.config.EnableEPSController && service.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeCluster { - lbService.Spec.Selector = vmiLabels - } else if lb.config.Selectorless == nil || !*lb.config.Selectorless { + if lb.config.EnableEPSController != nil && *lb.config.EnableEPSController && service.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeLocal { + lbService.Spec.Selector = nil + } else if lb.config.Selectorless != nil && *lb.config.Selectorless { + lbService.Spec.Selector = nil + } else { lbService.Spec.Selector = vmiLabels } if len(service.Spec.ExternalIPs) > 0 {