diff --git a/README.md b/README.md index f9cc57dc..5a39ee1b 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,7 @@ These annotations are deprecated, and will be removed in a future release. Annotation (Suffix) | Values | Default | Description | Scheduled Removal ---|---|---|---|--- -`protocol` | `tcp`, `http`, `https` | `tcp` | This annotation is used to specify the default protocol for Linode NodeBalancer. For ports specified in the `linode-loadbalancer-tls-ports` annotation, this protocol is overwritten to `https` | Q4 2020 `proxy-protcol` | `none`, `v1`, `v2` | `none` | Specifies whether to use a version of Proxy Protocol on the underlying NodeBalancer | Q4 2021 -`tls` | json array (e.g. `[ { "tls-secret-name": "prod-app-tls", "port": 443}, {"tls-secret-name": "dev-app-tls", "port": 8443} ]`) | | Specifies TLS ports with their corresponding secrets, the secret type should be `kubernetes.io/tls | Q4 2020 #### Annotation bool values diff --git a/cloud/linode/loadbalancers.go b/cloud/linode/loadbalancers.go index 23a213bb..5c0804d4 100644 --- a/cloud/linode/loadbalancers.go +++ b/cloud/linode/loadbalancers.go @@ -647,7 +647,7 @@ func getPortConfig(service *v1.Service, port int) (portConfig, error) { proxyProtocol := portConfigAnnotation.ProxyProtocol if proxyProtocol == "" { var ok bool - for _, ann := range []string{annLinodeDefaultProxyProtocol, annLinodeProxyProtocol} { + for _, ann := range []string{annLinodeDefaultProxyProtocol, annLinodeProxyProtocolDeprecated} { proxyProtocol, ok = service.Annotations[ann] if ok { break @@ -688,13 +688,14 @@ func getHealthCheckType(service *v1.Service) (linodego.ConfigCheck, error) { } func getPortConfigAnnotation(service *v1.Service, port int) (portConfigAnnotation, error) { + annotation := portConfigAnnotation{} annotationKey := annLinodePortConfigPrefix + strconv.Itoa(port) annotationJSON, ok := service.Annotations[annotationKey] + if !ok { - return tryDeprecatedTLSAnnotation(service, port) + return annotation, nil } - annotation := portConfigAnnotation{} err := json.Unmarshal([]byte(annotationJSON), &annotation) if err != nil { return annotation, err diff --git a/cloud/linode/loadbalancers_deprecated.go b/cloud/linode/loadbalancers_deprecated.go index c9047aa0..0fb1ad33 100644 --- a/cloud/linode/loadbalancers_deprecated.go +++ b/cloud/linode/loadbalancers_deprecated.go @@ -1,52 +1,5 @@ package linode -import ( - "encoding/json" - - v1 "k8s.io/api/core/v1" -) - const ( - annLinodeProtocolDeprecated = "service.beta.kubernetes.io/linode-loadbalancer-protocol" - annLinodeLoadBalancerTLSDeprecated = "service.beta.kubernetes.io/linode-loadbalancer-tls" - annLinodeProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol" + annLinodeProxyProtocolDeprecated = "service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol" ) - -type tlsAnnotationDeprecated struct { - TLSSecretName string `json:"tls-secret-name"` - Port int `json:"port"` -} - -func tryDeprecatedTLSAnnotation(service *v1.Service, port int) (portConfigAnnotation, error) { - annotation := portConfigAnnotation{} - tlsAnnotation, err := getTLSAnnotationDeprecated(service, port) - if err != nil { - return annotation, err - } - - if tlsAnnotation != nil { - annotation.Protocol = "https" - annotation.TLSSecretName = tlsAnnotation.TLSSecretName - } else if protocol, ok := service.Annotations[annLinodeProtocolDeprecated]; ok { - annotation.Protocol = protocol - } - return annotation, nil -} - -func getTLSAnnotationDeprecated(service *v1.Service, port int) (*tlsAnnotationDeprecated, error) { - annotationJSON, ok := service.Annotations[annLinodeLoadBalancerTLSDeprecated] - if !ok { - return nil, nil - } - tlsAnnotations := make([]*tlsAnnotationDeprecated, 0) - err := json.Unmarshal([]byte(annotationJSON), &tlsAnnotations) - if err != nil { - return nil, err - } - for _, tlsAnnotation := range tlsAnnotations { - if tlsAnnotation.Port == port { - return tlsAnnotation, nil - } - } - return nil, nil -} diff --git a/cloud/linode/loadbalancers_deprecated_test.go b/cloud/linode/loadbalancers_deprecated_test.go deleted file mode 100644 index 576aa6bb..00000000 --- a/cloud/linode/loadbalancers_deprecated_test.go +++ /dev/null @@ -1,539 +0,0 @@ -package linode - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - - "reflect" - "testing" - - "github.com/linode/linodego" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestCCMLoadBalancersDeprecated(t *testing.T) { - fake := newFake(t) - ts := httptest.NewServer(fake) - defer ts.Close() - - linodeClient := linodego.NewClient(http.DefaultClient) - linodeClient.SetBaseURL(ts.URL) - - testCases := []struct { - name string - f func(*testing.T, *linodego.Client) - }{ - { - name: "Get Load Balancer", - f: testGetLoadBalancerDeprecated, - }, - { - name: "Build Load Balancer Request", - f: testBuildLoadBalancerRequestDeprecated, - }, - { - name: "Ensure Load Balancer Deleted", - f: testEnsureLoadBalancerDeletedDeprecated, - }, - { - name: "Ensure Load Balancer", - f: testEnsureLoadBalancerDeprecated, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - tc.f(t, &linodeClient) - }) - } -} - -func testBuildLoadBalancerRequestDeprecated(t *testing.T, client *linodego.Client) { - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - UID: "foobar123", - Annotations: map[string]string{ - annLinodeProtocolDeprecated: "tcp", - }, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "test", - Protocol: "TCP", - Port: int32(80), - NodePort: int32(30000), - }, - }, - }, - } - nodes := []*v1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-3", - }, - }, - } - - lb := &loadbalancers{client, "us-west", nil} - nb, err := lb.buildLoadBalancerRequest(context.TODO(), "linodelb", svc, nodes) - if err != nil { - t.Fatal(err) - } - - if nb == nil { - t.Error("unexpected nodeID") - t.Logf("expected: != \"\"") - t.Logf("actual: %v", lb) - } - if !reflect.DeepEqual(err, err) { - t.Error("unexpected error") - t.Logf("expected: %v", nil) - t.Logf("actual: %v", err) - } - - configs, err := client.ListNodeBalancerConfigs(context.TODO(), nb.ID, nil) - if err != nil { - t.Fatal(err) - } - - if len(configs) != len(svc.Spec.Ports) { - t.Error("unexpected nodebalancer config count") - t.Logf("expected: %v", len(svc.Spec.Ports)) - t.Logf("actual: %v", len(configs)) - } - - nbNodes, _ := client.ListNodeBalancerNodes(context.TODO(), nb.ID, configs[0].ID, nil) - - if len(nbNodes) != len(nodes) { - t.Error("unexpected nodebalancer nodes count") - t.Logf("expected: %v", len(nodes)) - t.Logf("actual: %v", len(nbNodes)) - } - -} - -func testEnsureLoadBalancerDeletedDeprecated(t *testing.T, client *linodego.Client) { - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - UID: "foobar123", - Annotations: map[string]string{ - annLinodeProtocolDeprecated: "tcp", - }, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "test", - Protocol: "TCP", - Port: int32(80), - NodePort: int32(30000), - }, - }, - }, - } - testcases := []struct { - name string - clusterName string - service *v1.Service - err error - }{ - { - "load balancer delete", - "linodelb", - svc, - nil, - }, - { - "load balancer not exists", - "linodelb", - &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "notexists", - UID: "notexists123", - Annotations: map[string]string{ - annLinodeProtocolDeprecated: "tcp", - }, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "test", - Protocol: "TCP", - Port: int32(80), - NodePort: int32(30000), - }, - }, - }, - }, - nil, - }, - } - - lb := &loadbalancers{client, "us-west", nil} - configs := []*linodego.NodeBalancerConfigCreateOptions{} - _, err := lb.createNodeBalancer(context.TODO(), "linodelb", svc, configs) - if err != nil { - t.Fatal(err) - } - defer func() { _ = lb.EnsureLoadBalancerDeleted(context.TODO(), "linodelb", svc) }() - - for _, test := range testcases { - t.Run(test.name, func(t *testing.T) { - err := lb.EnsureLoadBalancerDeleted(context.TODO(), test.clusterName, test.service) - if !reflect.DeepEqual(err, test.err) { - t.Error("unexpected error") - t.Logf("expected: %v", test.err) - t.Logf("actual: %v", err) - } - }) - } -} - -func testEnsureLoadBalancerDeprecated(t *testing.T, client *linodego.Client) { - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testensure", - UID: "foobar123", - Annotations: map[string]string{ - annLinodeProtocolDeprecated: "tcp", - }, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "test", - Protocol: "TCP", - Port: int32(8000), - NodePort: int32(30000), - }, - { - Name: "test2", - Protocol: "TCP", - Port: int32(80), - NodePort: int32(30001), - }, - }, - }, - } - - lb := &loadbalancers{client, "us-west", nil} - - configs := []*linodego.NodeBalancerConfigCreateOptions{} - nb, err := lb.createNodeBalancer(context.TODO(), "linodelb", svc, configs) - if err != nil { - t.Fatal(err) - } - - svc.Status.LoadBalancer = *makeLoadBalancerStatus(nb) - defer func() { _ = lb.EnsureLoadBalancerDeleted(context.TODO(), "linodelb", svc) }() - lbStatus, exists, err := lb.GetLoadBalancer(context.TODO(), "linodelb", svc) - if err != nil { - t.Fatal(err) - } - if !exists { - t.Fatal("Node balancer not found") - } - - testcases := []struct { - name string - service *v1.Service - nodes []*v1.Node - clusterName string - nbIP string - err error - }{ - { - "update load balancer", - svc, - []*v1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{ - { - Type: v1.NodeInternalIP, - Address: "127.0.0.1", - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{ - { - Type: v1.NodeInternalIP, - Address: "127.0.0.2", - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-3", - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{ - { - Type: v1.NodeInternalIP, - Address: "127.0.0.3", - }, - }, - }, - }, - }, - "linodelb", - lbStatus.Ingress[0].IP, - nil, - }, - } - - for _, test := range testcases { - t.Run(test.name, func(t *testing.T) { - lbStatus, err := lb.EnsureLoadBalancer(context.TODO(), test.clusterName, test.service, test.nodes) - if err != nil { - t.Fatal(err) - } - if lbStatus.Ingress[0].IP != test.nbIP { - t.Error("unexpected error") - t.Logf("expected: %v", test.nbIP) - t.Logf("actual: %v", lbStatus.Ingress) - } - if !reflect.DeepEqual(err, test.err) { - t.Error("unexpected error") - t.Logf("expected: %v", test.err) - t.Logf("actual: %v", err) - } - }) - } -} - -func testGetLoadBalancerDeprecated(t *testing.T, client *linodego.Client) { - lb := &loadbalancers{client, "us-west", nil} - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - UID: "foobar123", - Annotations: map[string]string{ - annLinodeProtocolDeprecated: "tcp", - }, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "test", - Protocol: "TCP", - Port: int32(80), - NodePort: int32(30000), - }, - }, - }, - } - - configs := []*linodego.NodeBalancerConfigCreateOptions{} - nb, err := lb.createNodeBalancer(context.TODO(), "linodelb", svc, configs) - if err != nil { - t.Fatal(err) - } - - lbStatus := makeLoadBalancerStatus(nb) - svc.Status.LoadBalancer = *lbStatus - defer func() { _ = lb.EnsureLoadBalancerDeleted(context.TODO(), "linodelb", svc) }() - testcases := []struct { - name string - service *v1.Service - clusterName string - found bool - err error - }{ - { - "Load balancer exists", - svc, - "linodelb", - true, - nil, - }, - { - "Load balancer not exists", - - &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "notexists", - UID: "notexists123", - Annotations: map[string]string{ - annLinodeProtocolDeprecated: "tcp", - }, - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "test", - Protocol: "TCP", - Port: int32(80), - NodePort: int32(30000), - }, - }, - }, - }, - "linodelb", - false, - nil, - }, - } - - for _, test := range testcases { - t.Run(test.name, func(t *testing.T) { - - _, found, err := lb.GetLoadBalancer(context.TODO(), test.clusterName, test.service) - if found != test.found { - t.Error("unexpected error") - t.Logf("expected: %v", test.found) - t.Logf("actual: %v", found) - } - if !reflect.DeepEqual(err, test.err) { - t.Error("unexpected error") - t.Logf("expected: %v", test.err) - t.Logf("actual: %v", err) - } - }) - } -} - -func Test_getTLSAnnotationsDeprecated(t *testing.T) { - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "test", - Protocol: "TCP", - Port: int32(80), - NodePort: int32(30000), - }, - }, - }, - } - - testcases := []struct { - name string - ann map[string]string - annTLS *tlsAnnotationDeprecated - err error - }{ - { - name: "Test single TLS annotation", - ann: map[string]string{annLinodeLoadBalancerTLSDeprecated: `[ { "tls-secret-name": "prod-app-tls", "port": 443} ]`}, - annTLS: &tlsAnnotationDeprecated{ - TLSSecretName: "prod-app-tls", - Port: 443, - }, - err: nil, - }, - { - name: "Test multiple TLS annotation", - ann: map[string]string{annLinodeLoadBalancerTLSDeprecated: `[ { "tls-secret-name": "prod-app-tls", "port": 443}, {"tls-secret-name": "dev-app-tls", "port": 8443} ]`}, - annTLS: &tlsAnnotationDeprecated{ - TLSSecretName: "prod-app-tls", - Port: 443, - }, - err: nil, - }, - { - name: "Test invalid json", - ann: map[string]string{annLinodeLoadBalancerTLSDeprecated: `[ { "tls-secret-name": "prod-app-tls", "port": 443}`}, - annTLS: nil, - err: json.Unmarshal([]byte(`[ { "tls-secret-name": "prod-app-tls", "port": 443}`), &tlsAnnotationDeprecated{}), - }, - } - for _, test := range testcases { - t.Run(test.name, func(t *testing.T) { - svc.Annotations = test.ann - ann, err := getTLSAnnotationDeprecated(svc, 443) - if !reflect.DeepEqual(ann, test.annTLS) { - t.Error("unexpected error") - t.Logf("expected: %v", test.annTLS) - t.Logf("actual: %v", ann) - } - if !reflect.DeepEqual(err, test.err) { - t.Error("unexpected error") - t.Logf("expected: %v", test.err) - t.Logf("actual: %v", err) - } - }) - } -} - -func Test_tryDeprecatedTLSAnnotation(t *testing.T) { - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - } - testcases := []struct { - name string - ann map[string]string - expectedProtocol string - expectedTLSSecretName string - }{ - { - name: "Test TLS annotation for port in JSON", - ann: map[string]string{annLinodeLoadBalancerTLSDeprecated: `[ { "tls-secret-name": "prod-app-tls", "port": 443} ]`}, - expectedProtocol: "https", - expectedTLSSecretName: "prod-app-tls", - }, - { - name: "Test Linode Protocol set as default", - ann: map[string]string{annLinodeProtocolDeprecated: `https`}, - expectedProtocol: "https", - expectedTLSSecretName: "", - }, - { - name: "Test Linode Protocol when both set", - ann: map[string]string{ - annLinodeLoadBalancerTLSDeprecated: `[ { "tls-secret-name": "prod-app-tls", "port": 443} ]`, - annLinodeProtocolDeprecated: `tcp`, - }, - expectedProtocol: "https", - expectedTLSSecretName: "prod-app-tls", - }, - } - - for _, test := range testcases { - t.Run(test.name, func(t *testing.T) { - svc.Annotations = test.ann - portConfigAnnotation, _ := tryDeprecatedTLSAnnotation(svc, 443) - if portConfigAnnotation.Protocol != test.expectedProtocol { - t.Error("unexpected error") - t.Logf("expected: %v", test.expectedProtocol) - t.Logf("actual: %v", portConfigAnnotation.Protocol) - } - if portConfigAnnotation.TLSSecretName != test.expectedTLSSecretName { - t.Error("unexpected error") - t.Logf("expected: %v", test.expectedTLSSecretName) - t.Logf("actual: %v", portConfigAnnotation.TLSSecretName) - } - }) - } -} diff --git a/e2e/test/ccm_e2e_deprecated_test.go b/e2e/test/ccm_e2e_deprecated_test.go deleted file mode 100644 index 0ce56ac3..00000000 --- a/e2e/test/ccm_e2e_deprecated_test.go +++ /dev/null @@ -1,812 +0,0 @@ -package test - -import ( - "e2e_test/test/framework" - "io/ioutil" - "log" - "net/http" - "strconv" - "strings" - - "github.com/appscode/go/wait" - "github.com/codeskyblue/go-sh" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - core "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -var _ = Describe("CloudControllerManagerDeprecated", func() { - var ( - err error - f *framework.Invocation - workers []string - ) - - const ( - annLinodeLoadBalancerTLS = "service.beta.kubernetes.io/linode-loadbalancer-tls" - annLinodeProtocol = "service.beta.kubernetes.io/linode-loadbalancer-protocol" - annLinodeHealthCheckType = "service.beta.kubernetes.io/linode-loadbalancer-check-type" - annLinodeCheckBody = "service.beta.kubernetes.io/linode-loadbalancer-check-body" - annLinodeCheckPath = "service.beta.kubernetes.io/linode-loadbalancer-check-path" - annLinodeHealthCheckInterval = "service.beta.kubernetes.io/linode-loadbalancer-check-interval" - annLinodeHealthCheckTimeout = "service.beta.kubernetes.io/linode-loadbalancer-check-timeout" - annLinodeHealthCheckAttempts = "service.beta.kubernetes.io/linode-loadbalancer-check-attempts" - annLinodeHealthCheckPassive = "service.beta.kubernetes.io/linode-loadbalancer-check-passive" - ) - - BeforeEach(func() { - f = root.Invoke() - workers, err = f.GetNodeList() - Expect(err).NotTo(HaveOccurred()) - Expect(len(workers)).Should(BeNumerically(">=", 2)) - }) - - var createPodWithLabel = func(pods []string, ports []core.ContainerPort, image string, labels map[string]string, selectNode bool) { - for i, pod := range pods { - p := f.LoadBalancer.GetPodObject(pod, image, ports, labels) - if selectNode { - p = f.LoadBalancer.SetNodeSelector(p, workers[i]) - } - err = f.LoadBalancer.CreatePod(p) - Expect(err).NotTo(HaveOccurred()) - } - } - - var deletePods = func(pods []string) { - for _, pod := range pods { - err = f.LoadBalancer.DeletePod(pod) - Expect(err).NotTo(HaveOccurred()) - } - } - - var deleteService = func() { - err = f.LoadBalancer.DeleteService() - Expect(err).NotTo(HaveOccurred()) - } - - var deleteSecret = func(name string) { - err = f.LoadBalancer.DeleteSecret(name) - Expect(err).NotTo(HaveOccurred()) - } - - var createServiceWithSelector = func(selector map[string]string, ports []core.ServicePort, isSessionAffinityClientIP bool) { - err = f.LoadBalancer.CreateService(selector, nil, ports, isSessionAffinityClientIP) - Expect(err).NotTo(HaveOccurred()) - } - - var createServiceWithAnnotations = func(labels map[string]string, annotations map[string]string, ports []core.ServicePort, isSessionAffinityClientIP bool) { - err = f.LoadBalancer.CreateService(labels, annotations, ports, isSessionAffinityClientIP) - Expect(err).NotTo(HaveOccurred()) - } - - var getResponseFromSamePod = func(link string) { - var oldResp, newResp string - Eventually(func() string { - resp, err := http.Get(link) - if err == nil { - byteData, _ := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - oldResp = string(byteData) - } - - return oldResp - }).ShouldNot(Equal("")) - - for i := 0; i <= 10; i++ { - resp, err := http.Get(link) - if err == nil { - byteData, _ := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - newResp = string(byteData) - log.Println(newResp) - } - } - } - - var checkNumberOfUpNodes = func(numNodes int) { - By("Checking the Number of Up Nodes") - Eventually(func() int { - nbConfig, err := f.LoadBalancer.GetNodeBalancerConfig(framework.TestServerResourceName) - Expect(err).NotTo(HaveOccurred()) - return nbConfig.NodesStatus.Up - }).Should(Equal(numNodes)) - } - - var checkNodeBalancerConfig = func(checkType, path, body, interval, timeout, attempts, checkPassive string) { - By("Getting NodeBalancer Configuration") - nbConfig, err := f.LoadBalancer.GetNodeBalancerConfig(framework.TestServerResourceName) - Expect(err).NotTo(HaveOccurred()) - - By("Checking Health Check Type") - Expect(string(nbConfig.Check) == checkType).Should(BeTrue()) - - if path != "" { - By("Checking Health Check Path") - Expect(nbConfig.CheckPath == path).Should(BeTrue()) - } - - if body != "" { - By("Checking Health Check Body") - Expect(nbConfig.CheckBody == body).Should(BeTrue()) - } - - if interval != "" { - By("Checking TCP Connection Health Check Body") - intInterval, err := strconv.Atoi(interval) - Expect(err).NotTo(HaveOccurred()) - - Expect(nbConfig.CheckInterval == intInterval).Should(BeTrue()) - } - - if timeout != "" { - By("Checking TCP Connection Health Check Timeout") - intTimeout, err := strconv.Atoi(timeout) - Expect(err).NotTo(HaveOccurred()) - - Expect(nbConfig.CheckTimeout == intTimeout).Should(BeTrue()) - } - - if attempts != "" { - By("Checking TCP Connection Health Check Attempts") - intAttempts, err := strconv.Atoi(attempts) - Expect(err).NotTo(HaveOccurred()) - - Expect(nbConfig.CheckAttempts == intAttempts).Should(BeTrue()) - } - - if checkPassive != "" { - By("Checking for Passive Health Check") - boolCheckPassive, err := strconv.ParseBool(checkPassive) - Expect(err).NotTo(HaveOccurred()) - - Expect(nbConfig.CheckPassive == boolCheckPassive).Should(BeTrue()) - } - - checkNumberOfUpNodes(2) - } - - var addNewNode = func() { - _, err := sh.Command("terraform", "apply", "-var", "nodes=3", "-auto-approve").Output() - Expect(err).NotTo(HaveOccurred()) - } - - var deleteNewNode = func() { - _, err := sh.Command("terraform", "apply", "-var", "nodes=2", "-auto-approve").Output() - Expect(err).NotTo(HaveOccurred()) - } - - var waitForNodeAddition = func() { - checkNumberOfUpNodes(3) - } - - Describe("Test", func() { - Context("Simple", func() { - Context("Load Balancer", func() { - var ( - pods []string - labels map[string]string - ) - - BeforeEach(func() { - pods = []string{"test-pod-1", "test-pod-2"} - ports := []core.ContainerPort{ - { - Name: "http-1", - ContainerPort: 8080, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http-1", - Port: 80, - TargetPort: intstr.FromInt(8080), - Protocol: "TCP", - }, - } - labels = map[string]string{ - "app": "test-loadbalancer", - } - - By("Creating Pods") - createPodWithLabel(pods, ports, framework.TestServerImage, labels, true) - - By("Creating Service") - createServiceWithSelector(labels, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - }) - - It("should reach all pods", func() { - By("Checking TCP Response") - eps, err := f.LoadBalancer.GetHTTPEndpoints() - Expect(err).NotTo(HaveOccurred()) - Expect(len(eps)).Should(BeNumerically(">=", 1)) - - var counter1, counter2 int - - By("Waiting for Response from the LoadBalancer url: " + eps[0]) - err = wait.PollImmediate(framework.RetryInterval, framework.RetryTimeout, func() (bool, error) { - resp, err := sh.Command("curl", "--max-time", "5", "-s", eps[0]).Output() - if err != nil { - return false, nil - } - stringResp := string(resp) - if strings.Contains(stringResp, pods[0]) { - log.Println("Got response from " + pods[0]) - counter1++ - } else if strings.Contains(stringResp, pods[1]) { - log.Println("Got response from " + pods[1]) - counter2++ - } - - if counter1 > 0 && counter2 > 0 { - return true, nil - } - return false, nil - }) - Expect(counter1).Should(BeNumerically(">", 0)) - Expect(counter2).Should(BeNumerically(">", 0)) - }) - }) - }) - }) - - Describe("Test", func() { - Context("LoadBalancer", func() { - Context("With single TLS port", func() { - var ( - pods []string - labels map[string]string - annotations map[string]string - secretName string - ) - BeforeEach(func() { - pods = []string{"test-single-port-pod"} - ports := []core.ContainerPort{ - { - Name: "https", - ContainerPort: 8080, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "https", - Port: 80, - TargetPort: intstr.FromInt(8080), - Protocol: "TCP", - }, - } - secretName = "tls-secret" - labels = map[string]string{ - "app": "test-loadbalancer", - } - annotations = map[string]string{ - annLinodeLoadBalancerTLS: `[ { "tls-secret-name": "` + secretName + `", "port": 80} ]`, - annLinodeProtocol: "https", - } - - By("Creating Pod") - createPodWithLabel(pods, ports, framework.TestServerImage, labels, false) - - By("Creating Secret") - err = f.LoadBalancer.CreateTLSSecret("tls-secret") - Expect(err).NotTo(HaveOccurred()) - - By("Creating Service") - createServiceWithAnnotations(labels, annotations, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Secrets") - deletePods(pods) - - By("Deleting the Service") - deleteService() - - By("Deleting the Secret") - deleteSecret(secretName) - }) - - It("should reach the pod via tls", func() { - By("Checking TCP Response") - eps, err := f.LoadBalancer.GetHTTPEndpoints() - Expect(err).NotTo(HaveOccurred()) - Expect(len(eps)).Should(BeNumerically(">=", 1)) - - By("Waiting for Response from the LoadBalancer url: " + eps[0]) - err = framework.WaitForHTTPSResponse(eps[0], pods[0]) - Expect(err).NotTo(HaveOccurred()) - }) - }) - - Context("With Multiple TLS Ports", func() { - var ( - pods []string - labels map[string]string - annotations map[string]string - secretName1 string - secretName2 string - ) - BeforeEach(func() { - pods = []string{"tls-multi-port-pod"} - secretName1 = "tls-secret-1" - secretName2 = "tls-secret-2" - labels = map[string]string{ - "app": "test-loadbalancer", - } - annotations = map[string]string{ - annLinodeLoadBalancerTLS: `[ { "tls-secret-name": "` + secretName1 + `", "port": 80}, {"tls-secret-name": "` + secretName2 + `", "port": 443}]`, - annLinodeProtocol: "https", - } - ports := []core.ContainerPort{ - { - Name: "https1", - ContainerPort: 8080, - }, - { - Name: "https2", - ContainerPort: 8989, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "https-1", - Port: 80, - TargetPort: intstr.FromInt(8080), - Protocol: "TCP", - }, - { - Name: "https-2", - Port: 443, - TargetPort: intstr.FromInt(8989), - Protocol: "TCP", - }, - } - - By("Creating Pod") - createPodWithLabel(pods, ports, framework.TestServerImage, labels, false) - - By("Creating Secret") - err = f.LoadBalancer.CreateTLSSecret(secretName1) - Expect(err).NotTo(HaveOccurred()) - err = f.LoadBalancer.CreateTLSSecret(secretName2) - Expect(err).NotTo(HaveOccurred()) - - By("Creating Service") - createServiceWithAnnotations(labels, annotations, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Secrets") - deletePods(pods) - - By("Deleting the Service") - deleteService() - - By("Deleting the Secret") - deleteSecret(secretName1) - deleteSecret(secretName2) - }) - - It("should reach the pod via tls", func() { - By("Checking TCP Response") - eps, err := f.LoadBalancer.GetHTTPEndpoints() - Expect(err).NotTo(HaveOccurred()) - Expect(len(eps)).Should(BeNumerically(">=", 1)) - - By("Waiting for Response from the LoadBalancer urls: " + eps[0] + ", " + eps[1]) - for _, ep := range eps { - err = framework.WaitForHTTPSResponse(ep, pods[0]) - Expect(err).NotTo(HaveOccurred()) - } - }) - }) - - Context("With Multiple HTTP Ports", func() { - var ( - pods []string - labels map[string]string - ) - - BeforeEach(func() { - pods = []string{"test-pod-http"} - ports := []core.ContainerPort{ - { - Name: "http-1", - ContainerPort: 8080, - }, - { - Name: "http-2", - ContainerPort: 8989, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http-1", - Port: 80, - TargetPort: intstr.FromInt(8080), - Protocol: "TCP", - }, - { - Name: "http-2", - Port: 8888, - TargetPort: intstr.FromInt(8989), - Protocol: "TCP", - }, - } - labels = map[string]string{ - "app": "test-loadbalancer", - } - - By("Creating Pods") - createPodWithLabel(pods, ports, framework.TestServerImage, labels, true) - - By("Creating Service") - createServiceWithSelector(labels, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - }) - - It("should reach all pods", func() { - By("Checking TCP Response") - eps, err := f.LoadBalancer.GetHTTPEndpoints() - Expect(err).NotTo(HaveOccurred()) - Expect(len(eps)).Should(BeNumerically(">=", 1)) - - By("Waiting for Response from the LoadBalancer url: " + eps[0] + " " + eps[1]) - for _, ep := range eps { - err = framework.WaitForHTTPResponse(ep, pods[0]) - Expect(err).NotTo(HaveOccurred()) - } - }) - }) - - Context("With SessionAffinity", func() { - var ( - pods []string - labels map[string]string - ) - - BeforeEach(func() { - pods = []string{"test-pod-1", "test-pod-2"} - ports := []core.ContainerPort{ - { - Name: "http-1", - ContainerPort: 8080, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http-1", - Port: 80, - TargetPort: intstr.FromInt(8080), - Protocol: "TCP", - }, - } - labels = map[string]string{ - "app": "test-loadbalancer", - } - - By("Creating Pods") - createPodWithLabel(pods, ports, framework.TestServerImage, labels, false) - - By("Creating Service") - createServiceWithSelector(labels, servicePorts, true) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - }) - - It("should reach the same pod every time it requests", func() { - By("Checking TCP Response") - eps, err := f.LoadBalancer.GetHTTPEndpoints() - Expect(err).NotTo(HaveOccurred()) - Expect(len(eps)).Should(BeNumerically(">=", 1)) - - By("Waiting for Response from the LoadBalancer url: " + eps[0]) - getResponseFromSamePod(eps[0]) - }) - }) - - Context("For HTTP body health check", func() { - var ( - pods []string - labels map[string]string - annotations map[string]string - - checkType = "http_body" - path = "/" - body = "nginx" - ) - BeforeEach(func() { - pods = []string{"test-pod-http-body"} - ports := []core.ContainerPort{ - { - Name: "http", - ContainerPort: 80, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http", - Port: 80, - TargetPort: intstr.FromInt(80), - Protocol: "TCP", - }, - } - - labels = map[string]string{ - "app": "test-loadbalancer", - } - annotations = map[string]string{ - annLinodeHealthCheckType: checkType, - annLinodeCheckPath: path, - annLinodeCheckBody: body, - annLinodeProtocol: "http", - } - - By("Creating Pod") - createPodWithLabel(pods, ports, "nginx", labels, false) - - By("Creating Service") - createServiceWithAnnotations(labels, annotations, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - }) - - It("should successfully check the health of 2 nodes", func() { - By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkType, path, body, "", "", "", "") - }) - }) - - Context("With Node Addition", func() { - var ( - pods []string - labels map[string]string - ) - - BeforeEach(func() { - pods = []string{"test-pod-node-add"} - ports := []core.ContainerPort{ - { - Name: "http-1", - ContainerPort: 8080, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http-1", - Port: 80, - TargetPort: intstr.FromInt(8080), - Protocol: "TCP", - }, - } - labels = map[string]string{ - "app": "test-loadbalancer", - } - - By("Creating Pods") - createPodWithLabel(pods, ports, framework.TestServerImage, labels, false) - - By("Creating Service") - createServiceWithSelector(labels, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - - By("Deleting the Newly Created Nodes") - deleteNewNode() - }) - - It("should reach the same pod every time it requests", func() { - By("Adding a New Node") - addNewNode() - - By("Waiting for the Node to be Added to the NodeBalancer") - waitForNodeAddition() - }) - }) - - Context("For TCP Connection health check", func() { - var ( - pods []string - labels map[string]string - annotations map[string]string - - checkType = "connection" - interval = "10" - timeout = "5" - attempts = "4" - ) - BeforeEach(func() { - pods = []string{"test-pod-tcp"} - ports := []core.ContainerPort{ - { - Name: "http", - ContainerPort: 80, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http", - Port: 80, - TargetPort: intstr.FromInt(80), - Protocol: "TCP", - }, - } - - labels = map[string]string{ - "app": "test-loadbalancer", - } - annotations = map[string]string{ - annLinodeHealthCheckType: checkType, - annLinodeProtocol: "tcp", - annLinodeHealthCheckInterval: interval, - annLinodeHealthCheckTimeout: timeout, - annLinodeHealthCheckAttempts: attempts, - } - - By("Creating Pod") - createPodWithLabel(pods, ports, "nginx", labels, false) - - By("Creating Service") - createServiceWithAnnotations(labels, annotations, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - }) - - It("should successfully check the health of 2 nodes", func() { - By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkType, "", "", interval, timeout, attempts, "") - }) - }) - - Context("For Passive Health Check", func() { - var ( - pods []string - labels map[string]string - annotations map[string]string - - checkType = "none" - checkPassive = "true" - ) - BeforeEach(func() { - pods = []string{"test-pod-passive-hc"} - ports := []core.ContainerPort{ - { - Name: "http", - ContainerPort: 80, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http", - Port: 80, - TargetPort: intstr.FromInt(80), - Protocol: "TCP", - }, - } - - labels = map[string]string{ - "app": "test-loadbalancer", - } - annotations = map[string]string{ - annLinodeHealthCheckPassive: checkPassive, - annLinodeHealthCheckType: checkType, - } - - By("Creating Pod") - createPodWithLabel(pods, ports, "nginx", labels, false) - - By("Creating Service") - createServiceWithAnnotations(labels, annotations, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - }) - - It("should successfully check the health of 2 nodes", func() { - By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkType, "", "", "", "", "", checkPassive) - }) - }) - - Context("For HTTP Status Health Check", func() { - var ( - pods []string - labels map[string]string - annotations map[string]string - - checkType = "http" - path = "/" - ) - BeforeEach(func() { - pods = []string{"test-pod-http-status"} - ports := []core.ContainerPort{ - { - Name: "http", - ContainerPort: 80, - }, - } - servicePorts := []core.ServicePort{ - { - Name: "http", - Port: 80, - TargetPort: intstr.FromInt(80), - Protocol: "TCP", - }, - } - - labels = map[string]string{ - "app": "test-loadbalancer", - } - annotations = map[string]string{ - annLinodeHealthCheckType: checkType, - annLinodeCheckPath: path, - annLinodeProtocol: "http", - } - - By("Creating Pod") - createPodWithLabel(pods, ports, "nginx", labels, false) - - By("Creating Service") - createServiceWithAnnotations(labels, annotations, servicePorts, false) - }) - - AfterEach(func() { - By("Deleting the Pods") - deletePods(pods) - - By("Deleting the Service") - deleteService() - }) - - It("should successfully check the health of 2 nodes", func() { - By("Checking NodeBalancer Configurations") - checkNodeBalancerConfig(checkType, path, "", "", "", "", "") - }) - }) - }) - }) -}) diff --git a/e2e/test/ccm_e2e_test.go b/e2e/test/ccm_e2e_test.go index a8a25e18..946b9075 100644 --- a/e2e/test/ccm_e2e_test.go +++ b/e2e/test/ccm_e2e_test.go @@ -27,19 +27,19 @@ var _ = Describe("e2e tests", func() { ) const ( - annLinodeProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol" - annLinodeDefaultProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol" - annLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol" - annLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-" - annLinodeLoadBalancerPreserve = "service.beta.kubernetes.io/linode-loadbalancer-preserve" - annLinodeHealthCheckType = "service.beta.kubernetes.io/linode-loadbalancer-check-type" - annLinodeCheckBody = "service.beta.kubernetes.io/linode-loadbalancer-check-body" - annLinodeCheckPath = "service.beta.kubernetes.io/linode-loadbalancer-check-path" - annLinodeHealthCheckInterval = "service.beta.kubernetes.io/linode-loadbalancer-check-interval" - annLinodeHealthCheckTimeout = "service.beta.kubernetes.io/linode-loadbalancer-check-timeout" - annLinodeHealthCheckAttempts = "service.beta.kubernetes.io/linode-loadbalancer-check-attempts" - annLinodeHealthCheckPassive = "service.beta.kubernetes.io/linode-loadbalancer-check-passive" - annLinodeNodeBalancerID = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id" + annLinodeProxyProtocolDeprecated = "service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol" + annLinodeDefaultProxyProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol" + annLinodeDefaultProtocol = "service.beta.kubernetes.io/linode-loadbalancer-default-protocol" + annLinodePortConfigPrefix = "service.beta.kubernetes.io/linode-loadbalancer-port-" + annLinodeLoadBalancerPreserve = "service.beta.kubernetes.io/linode-loadbalancer-preserve" + annLinodeHealthCheckType = "service.beta.kubernetes.io/linode-loadbalancer-check-type" + annLinodeCheckBody = "service.beta.kubernetes.io/linode-loadbalancer-check-body" + annLinodeCheckPath = "service.beta.kubernetes.io/linode-loadbalancer-check-path" + annLinodeHealthCheckInterval = "service.beta.kubernetes.io/linode-loadbalancer-check-interval" + annLinodeHealthCheckTimeout = "service.beta.kubernetes.io/linode-loadbalancer-check-timeout" + annLinodeHealthCheckAttempts = "service.beta.kubernetes.io/linode-loadbalancer-check-attempts" + annLinodeHealthCheckPassive = "service.beta.kubernetes.io/linode-loadbalancer-check-passive" + annLinodeNodeBalancerID = "service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id" ) BeforeEach(func() { @@ -506,8 +506,8 @@ var _ = Describe("e2e tests", func() { It("default annotations can be used to apply ProxyProtocol to all NodeBalancerConfigs", func() { annotations := make(map[string]string) - By("By specifying ProxyProtocol v2 using the deprecated annotation " + annLinodeProxyProtocol) - annotations[annLinodeProxyProtocol] = proxyProtocolV2 + By("By specifying ProxyProtocol v2 using the deprecated annotation " + annLinodeProxyProtocolDeprecated) + annotations[annLinodeProxyProtocolDeprecated] = proxyProtocolV2 updateServiceWithAnnotations(labels, annotations, servicePorts, false) By("Checking NodeBalancerConfig for port 80 should have default ProxyProtocol v2")