From 954ac2d5988d4fe1919e170ff096ad32210705f4 Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Tue, 19 Mar 2024 23:37:17 +0000 Subject: [PATCH 1/2] add support to auto annotate nodes --- cloud/linode/cloud.go | 11 +++++--- cloud/linode/node_controller.go | 40 +++++++++++++++++++++++---- deploy/chart/templates/daemonset.yaml | 6 ++++ deploy/chart/values.yaml | 7 +++++ main.go | 3 ++ 5 files changed, 58 insertions(+), 9 deletions(-) diff --git a/cloud/linode/cloud.go b/cloud/linode/cloud.go index 5e9b0436..4b270556 100644 --- a/cloud/linode/cloud.go +++ b/cloud/linode/cloud.go @@ -3,6 +3,7 @@ package linode import ( "fmt" "io" + "net" "os" "github.com/linode/linodego" @@ -25,10 +26,12 @@ const ( // We expect it to be initialized with flags external to this package, likely in // main.go var Options struct { - KubeconfigFlag *pflag.Flag - LinodeGoDebug bool - EnableRouteController bool - VPCName string + KubeconfigFlag *pflag.Flag + LinodeGoDebug bool + EnableRouteController bool + VPCName string + LinodeNodePrivateSubnet net.IPNet + AutoAnnotateNode bool } type linodeCloud struct { diff --git a/cloud/linode/node_controller.go b/cloud/linode/node_controller.go index e77eb20f..05790dcc 100644 --- a/cloud/linode/node_controller.go +++ b/cloud/linode/node_controller.go @@ -2,6 +2,8 @@ package linode import ( "context" + "fmt" + "net" "net/http" "os" "strconv" @@ -127,8 +129,14 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { lastUpdate := s.LastMetadataUpdate(node.Name) - uuid, ok := node.Labels[annotations.AnnLinodeHostUUID] - if ok && time.Since(lastUpdate) < s.ttl { + uuid, foundLabel := node.Labels[annotations.AnnLinodeHostUUID] + configuredPrivateIP, foundAnnotation := node.Annotations[annotations.AnnLinodeNodePrivateIP] + if foundLabel && foundAnnotation && time.Since(lastUpdate) < s.ttl { + return nil + } + + // if autoannotate is set to false, just check label and ttl + if foundLabel && !Options.AutoAnnotateNode && time.Since(lastUpdate) < s.ttl { return nil } @@ -138,7 +146,26 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { return err } - if uuid == linode.HostUUID { + expectedPrivateIP := "" + if Options.AutoAnnotateNode { + ips, err := s.instances.getLinodeIPv4Addresses(ctx, node) + if err != nil { + return fmt.Errorf("failed to get ips for linode: %w", err) + } + + if len(ips) == 0 { + return fmt.Errorf("no ips found for node %s", node.Name) + } + + for _, ip := range ips { + if Options.LinodeNodePrivateSubnet.Contains(net.ParseIP(ip.ip)) { + expectedPrivateIP = ip.ip + break + } + } + } + + if uuid == linode.HostUUID && configuredPrivateIP == expectedPrivateIP { s.SetLastMetadataUpdate(node.Name) return nil } @@ -150,13 +177,16 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { return err } - // It may be that the UUID has been set - if n.Labels[annotations.AnnLinodeHostUUID] == linode.HostUUID { + // It may be that the UUID label and private ip annotation has been set + if n.Labels[annotations.AnnLinodeHostUUID] == linode.HostUUID && n.Annotations[annotations.AnnLinodeNodePrivateIP] == expectedPrivateIP { return nil } // Try to update the node n.Labels[annotations.AnnLinodeHostUUID] = linode.HostUUID + if Options.AutoAnnotateNode { + n.Annotations[annotations.AnnLinodeNodePrivateIP] = expectedPrivateIP + } _, err = s.kubeclient.CoreV1().Nodes().Update(ctx, n, metav1.UpdateOptions{}) return err }); err != nil { diff --git a/deploy/chart/templates/daemonset.yaml b/deploy/chart/templates/daemonset.yaml index 4b2b3d41..8f4df2ba 100644 --- a/deploy/chart/templates/daemonset.yaml +++ b/deploy/chart/templates/daemonset.yaml @@ -45,6 +45,12 @@ spec: - --route-reconciliation-period={{ .Values.routeController.routeReconciliationPeriod }} {{- end }} {{- end }} + {{- if .Values.autoAnnotateNode }} + - --auto-annotate-node={{ .Values.autoAnnotateNode }} + {{- end }} + {{- if .Values.nodePrivateSubnet }} + - --linode-node-private-subnet={{ .Values.nodePrivateSubnet }} + {{- end }} volumeMounts: - mountPath: /etc/kubernetes name: k8s diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 2df40a14..11cb0a51 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -50,6 +50,13 @@ tolerations: # clusterCIDR: 10.0.0.0/8 # configureCloudRoutes: true +# This section enables auto-annotating the nodes for backend ip to use with nodebalancers +# By default, its set to false. +# autoAnnotateNode: false + +# Set only if one wants to change the default backend network(192.168.128.0/17) to be configured with nodebalancers +# nodePrivateSubnet: 192.168.128.0/17 + # This section adds the ability to pass environment variables to adjust CCM defaults # https://github.com/linode/linode-cloud-controller-manager/blob/master/cloud/linode/loadbalancers.go # LINODE_HOSTNAME_ONLY_INGRESS type bool is supported diff --git a/main.go b/main.go index dfcf10bd..2cac3d20 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "net" "os" "k8s.io/component-base/logs" @@ -78,6 +79,8 @@ func main() { command.Flags().BoolVar(&linode.Options.LinodeGoDebug, "linodego-debug", false, "enables debug output for the LinodeAPI wrapper") command.Flags().BoolVar(&linode.Options.EnableRouteController, "enable-route-controller", false, "enables route_controller for ccm") command.Flags().StringVar(&linode.Options.VPCName, "vpc-name", "", "vpc name whose routes will be managed by route-controller") + command.Flags().IPNetVar(&linode.Options.LinodeNodePrivateSubnet, "linode-node-private-subnet", net.IPNet{IP: net.ParseIP("192.168.128.0"), Mask: net.CIDRMask(17, 32)}, "specifies backend network to use for nodebalancers") + command.Flags().BoolVar(&linode.Options.AutoAnnotateNode, "auto-annotate-node", false, "enables automatically adding node annotation for ip in subnet specified in linode-node-private-subnet") // Set static flags command.Flags().VisitAll(func(fl *pflag.Flag) { From c6cd51fd7301c9c41da3d33ce53b4707e89f2ba7 Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Wed, 20 Mar 2024 18:32:59 +0000 Subject: [PATCH 2/2] remove flags, rely on private ip returned by linode API --- cloud/linode/cloud.go | 11 ++++----- cloud/linode/node_controller.go | 32 ++++++++------------------- deploy/chart/templates/daemonset.yaml | 6 ----- deploy/chart/values.yaml | 7 ------ main.go | 3 --- 5 files changed, 13 insertions(+), 46 deletions(-) diff --git a/cloud/linode/cloud.go b/cloud/linode/cloud.go index 4b270556..5e9b0436 100644 --- a/cloud/linode/cloud.go +++ b/cloud/linode/cloud.go @@ -3,7 +3,6 @@ package linode import ( "fmt" "io" - "net" "os" "github.com/linode/linodego" @@ -26,12 +25,10 @@ const ( // We expect it to be initialized with flags external to this package, likely in // main.go var Options struct { - KubeconfigFlag *pflag.Flag - LinodeGoDebug bool - EnableRouteController bool - VPCName string - LinodeNodePrivateSubnet net.IPNet - AutoAnnotateNode bool + KubeconfigFlag *pflag.Flag + LinodeGoDebug bool + EnableRouteController bool + VPCName string } type linodeCloud struct { diff --git a/cloud/linode/node_controller.go b/cloud/linode/node_controller.go index 05790dcc..461407c9 100644 --- a/cloud/linode/node_controller.go +++ b/cloud/linode/node_controller.go @@ -2,8 +2,6 @@ package linode import ( "context" - "fmt" - "net" "net/http" "os" "strconv" @@ -135,11 +133,6 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { return nil } - // if autoannotate is set to false, just check label and ttl - if foundLabel && !Options.AutoAnnotateNode && time.Since(lastUpdate) < s.ttl { - return nil - } - linode, err := s.instances.lookupLinode(ctx, node) if err != nil { klog.Infof("instance lookup error: %s", err.Error()) @@ -147,21 +140,14 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { } expectedPrivateIP := "" - if Options.AutoAnnotateNode { - ips, err := s.instances.getLinodeIPv4Addresses(ctx, node) - if err != nil { - return fmt.Errorf("failed to get ips for linode: %w", err) - } - - if len(ips) == 0 { - return fmt.Errorf("no ips found for node %s", node.Name) - } - - for _, ip := range ips { - if Options.LinodeNodePrivateSubnet.Contains(net.ParseIP(ip.ip)) { - expectedPrivateIP = ip.ip - break - } + // linode API response for linode will contain only one private ip + // if any private ip is configured. If it changes in future or linode + // supports other subnets with nodebalancer, this logic needs to be updated. + // https://www.linode.com/docs/api/linode-instances/#linode-view + for _, addr := range linode.IPv4 { + if addr.IsPrivate() { + expectedPrivateIP = addr.String() + break } } @@ -184,7 +170,7 @@ func (s *nodeController) handleNode(ctx context.Context, node *v1.Node) error { // Try to update the node n.Labels[annotations.AnnLinodeHostUUID] = linode.HostUUID - if Options.AutoAnnotateNode { + if expectedPrivateIP != "" { n.Annotations[annotations.AnnLinodeNodePrivateIP] = expectedPrivateIP } _, err = s.kubeclient.CoreV1().Nodes().Update(ctx, n, metav1.UpdateOptions{}) diff --git a/deploy/chart/templates/daemonset.yaml b/deploy/chart/templates/daemonset.yaml index 8f4df2ba..4b2b3d41 100644 --- a/deploy/chart/templates/daemonset.yaml +++ b/deploy/chart/templates/daemonset.yaml @@ -45,12 +45,6 @@ spec: - --route-reconciliation-period={{ .Values.routeController.routeReconciliationPeriod }} {{- end }} {{- end }} - {{- if .Values.autoAnnotateNode }} - - --auto-annotate-node={{ .Values.autoAnnotateNode }} - {{- end }} - {{- if .Values.nodePrivateSubnet }} - - --linode-node-private-subnet={{ .Values.nodePrivateSubnet }} - {{- end }} volumeMounts: - mountPath: /etc/kubernetes name: k8s diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 11cb0a51..2df40a14 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -50,13 +50,6 @@ tolerations: # clusterCIDR: 10.0.0.0/8 # configureCloudRoutes: true -# This section enables auto-annotating the nodes for backend ip to use with nodebalancers -# By default, its set to false. -# autoAnnotateNode: false - -# Set only if one wants to change the default backend network(192.168.128.0/17) to be configured with nodebalancers -# nodePrivateSubnet: 192.168.128.0/17 - # This section adds the ability to pass environment variables to adjust CCM defaults # https://github.com/linode/linode-cloud-controller-manager/blob/master/cloud/linode/loadbalancers.go # LINODE_HOSTNAME_ONLY_INGRESS type bool is supported diff --git a/main.go b/main.go index 2cac3d20..dfcf10bd 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "context" "flag" "fmt" - "net" "os" "k8s.io/component-base/logs" @@ -79,8 +78,6 @@ func main() { command.Flags().BoolVar(&linode.Options.LinodeGoDebug, "linodego-debug", false, "enables debug output for the LinodeAPI wrapper") command.Flags().BoolVar(&linode.Options.EnableRouteController, "enable-route-controller", false, "enables route_controller for ccm") command.Flags().StringVar(&linode.Options.VPCName, "vpc-name", "", "vpc name whose routes will be managed by route-controller") - command.Flags().IPNetVar(&linode.Options.LinodeNodePrivateSubnet, "linode-node-private-subnet", net.IPNet{IP: net.ParseIP("192.168.128.0"), Mask: net.CIDRMask(17, 32)}, "specifies backend network to use for nodebalancers") - command.Flags().BoolVar(&linode.Options.AutoAnnotateNode, "auto-annotate-node", false, "enables automatically adding node annotation for ip in subnet specified in linode-node-private-subnet") // Set static flags command.Flags().VisitAll(func(fl *pflag.Flag) {