From be5e7cb43f08c5b2fbd2d5e47067f7f0ab3234f9 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Wed, 18 Dec 2024 10:47:43 +0800 Subject: [PATCH] Reopen e2e security policy traffic --- test/e2e/framework.go | 205 ++++++++++-------- ...a-pod-selector-with-match-expressions.yaml | 15 +- .../manifest/testSecurityPolicy/busybox.yaml | 14 -- .../testSecurityPolicy/netcat-pod.yaml | 12 - .../rule-in-policy-applied-to.yaml | 4 +- .../rule-in-rule-applied-to-with-src.yaml | 2 +- .../rule-in-rule-applied-to.yaml | 4 +- .../rule-out-policy-applied-to-with-dst.yaml | 4 +- ...plied-to-with-dst-with-dup-port-multi.yaml | 6 +- ...ule-applied-to-with-dst-with-dup-port.yaml | 6 +- .../rule-out-rule-applied-to-with-dst.yaml | 4 +- ...e-applied-to-with-expression-selector.yaml | 6 +- test/e2e/nsx_security_policy_test.go | 170 ++++++++------- test/e2e/precreated_vpc_test.go | 18 +- test/e2e/vclient.go | 3 +- 15 files changed, 224 insertions(+), 249 deletions(-) delete mode 100644 test/e2e/manifest/testSecurityPolicy/busybox.yaml delete mode 100644 test/e2e/manifest/testSecurityPolicy/netcat-pod.yaml diff --git a/test/e2e/framework.go b/test/e2e/framework.go index b2ece8a53..477a14de6 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -3,8 +3,10 @@ package e2e import ( "bytes" "context" + "encoding/json" "fmt" "net" + "net/http" "net/url" "os/exec" "regexp" @@ -36,8 +38,9 @@ import ( var log = &logger.Log const ( - defaultTimeout = 200 * time.Second - PolicyAPI = "policy/api/v1" + createVCNamespaceEndpoint = "/api/vcenter/namespaces/instances/v2" + defaultTimeout = 300 * time.Second + PolicyAPI = "policy/api/v1" ) type Status int @@ -293,6 +296,90 @@ func (data *TestData) createNamespace(namespace string, mutators ...func(ns *cor return nil } +// createVCNamespace creates a VC namespace with the provided namespace. +func (data *TestData) createVCNamespace(namespace string) error { + err := testData.vcClient.startSession() + if err != nil { + return err + } + defer func() { + testData.vcClient.closeSession() + }() + + svID, _ := data.vcClient.getSupervisorID() + vcNamespace := &VCNamespaceCreateSpec{ + Supervisor: svID, + Namespace: namespace, + NetworkSpec: InstancesNetworkConfigInfo{ + NetworkProvider: "NSX_VPC", + VpcNetwork: InstancesVpcNetworkInfo{ + DefaultSubnetSize: 16, + }, + }, + } + dataJson, err := json.Marshal(vcNamespace) + if err != nil { + log.Error(err, "Unable convert vcNamespace object to json bytes", "namespace", namespace) + return fmt.Errorf("unable convert vcNamespace object to json bytes: %v", err) + } + request, err := data.vcClient.prepareRequest(http.MethodPost, createVCNamespaceEndpoint, dataJson) + if err != nil { + log.Error(err, "Failed to prepare http request with vcNamespace data", "namespace", namespace) + return fmt.Errorf("failed to parepare http request with vcNamespace data: %v", err) + } + if _, err = data.vcClient.handleRequest(request, nil); err != nil { + log.Error(err, "Failed to create VC namespace", "namespace", namespace) + return err + } + // wait for the namespace on k8s running + err = wait.PollUntilContextTimeout(context.TODO(), 10*time.Second, defaultTimeout, false, func(ctx context.Context) (done bool, err error) { + ns, err := data.clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + if err != nil { + log.Error(err, "Check namespace existence", "namespace", namespace) + return false, err + } + + for _, condition := range ns.Status.Conditions { + if condition.Type == "NamespaceNetworkReady" && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) + if err != nil { + testData.deleteVCNamespace(namespace) + } + return err +} + +// deleteVCNamespace deletes the provided VC namespace and waits for deletion to actually complete. +func (data *TestData) deleteVCNamespace(namespace string) error { + err := testData.vcClient.startSession() + if err != nil { + return err + } + defer func() { + testData.vcClient.closeSession() + }() + + _ = testData.vcClient.deleteNamespace(namespace) + // wait for the namespace on k8s terminating + err = wait.PollUntilContextTimeout(context.TODO(), 10*time.Second, defaultTimeout, false, func(ctx context.Context) (done bool, err error) { + ns, err := data.clientset.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Info("Namespace not found, it has been deleted", "namespace", namespace) + return true, nil + } + log.Error(err, "Check namespace existence", "namespace", namespace) + return false, err + } + log.Info("Waiting for namespace to be deleted", "namespace", namespace, "status phase", ns.Status.Phase) + return false, nil + }) + return err +} + // deleteNamespace deletes the provided namespace and waits for deletion to actually complete. func (data *TestData) deleteNamespace(namespace string, timeout time.Duration) error { var gracePeriodSeconds int64 @@ -422,41 +509,6 @@ func (data *TestData) podWaitForIPs(timeout time.Duration, name, namespace strin return ips, nil } -/* -// deploymentWaitForIPsOrNames polls the K8s apiServer until the specified Pod in deployment has an IP address -func (data *TestData) deploymentWaitForIPsOrNames(timeout time.Duration, namespace, deployment string) ([]string, []string, error) { - podIPStrings := sets.NewString() - var podNames []string - opt := metav1.ListOptions{ - LabelSelector: "deployment=" + deployment, - } - err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - if pods, err := data.clientset.CoreV1().Pods(namespace).List(context.TODO(), opt); err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, fmt.Errorf("error when getting Pod %v", err) - } else { - for _, p := range pods.Items { - if p.Status.Phase != corev1.PodRunning { - return false, nil - } else if p.Status.PodIP == "" { - return false, nil - } else { - podIPStrings.Insert(p.Status.PodIP) - podNames = append(podNames, p.Name) - } - } - return true, nil - } - }) - if err != nil { - return nil, nil, err - } - return podIPStrings.List(), podNames, nil -} -*/ - func parsePodIPs(podIPStrings sets.Set[string]) (*PodIPs, error) { ips := new(PodIPs) for podIP := range podIPStrings { @@ -520,39 +572,6 @@ func (data *TestData) runCommandFromPod(namespace string, podName string, contai return stdoutB.String(), stderrB.String(), nil } -/* -func (data *TestData) runPingCommandFromPod(namespace string, podName string, targetPodIPs *PodIPs, count int) error { - var cmd []string - if targetPodIPs.ipv4 != nil { - cmd = []string{"ping", "-c", strconv.Itoa(count), targetPodIPs.ipv4.String()} - if _, _, err := data.runCommandFromPod(namespace, podName, podName, cmd); err != nil { - return err - } - } - if targetPodIPs.ipv6 != nil { - cmd = []string{"ping", "-6", "-c", strconv.Itoa(count), targetPodIPs.ipv6.String()} - if _, _, err := data.runCommandFromPod(namespace, podName, podName, cmd); err != nil { - return err - } - } - return nil -} - -func (data *TestData) runNetcatCommandFromPod(namespace string, podName string, containerName string, server string, port int) error { - cmd := []string{ - "/bin/sh", - "-c", - fmt.Sprintf("for i in $(seq 1 5); do nc -w 4 %s %d && exit 0 || sleep 1; done; exit 1", - server, port), - } - _, _, err := data.runCommandFromPod(namespace, podName, containerName, cmd) - if err != nil { - return err - } - return nil -} -*/ - func applyYAML(filename string, ns string) error { cmd := fmt.Sprintf("kubectl apply -f %s -n %s", filename, ns) if ns == "" { @@ -575,31 +594,6 @@ func applyYAML(filename string, ns string) error { return nil } -// Temporarily disable traffic check -/* -func runCommand(cmd string) (string, error) { - err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, defaultTimeout, false, func(ctx context.Context) (bool, error) { - var stdout, stderr bytes.Buffer - command := exec.Command("bash", "-c", cmd) - log.Info("Running command %s", cmd) - command.Stdout = &stdout - command.Stderr = &stderr - err := command.Run() - if err != nil { - log.Info("Error when running command %s: %v", cmd, err) - return false, nil - } - outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) - log.Info("Command %s returned with output: '%s' and error: '%s'", cmd, outStr, errStr) - if errStr != "" { - return false, nil - } - return true, nil - }) - return "", err -} -*/ - func deleteYAML(filename string, ns string) error { cmd := fmt.Sprintf("kubectl delete -f %s -n %s", filename, ns) if ns == "" { @@ -815,3 +809,24 @@ func (data *TestData) deleteService(nsName string, svcName string) error { func (data *TestData) useWCPSetup() bool { return data.vcClient != nil } + +func checkTrafficByCurl(ns, podname, containername, ip string, port int32, interval, timeout time.Duration) error { + // Test traffic from client Pod to server Pod + url := fmt.Sprintf("http://%s:%d", ip, port) + cmd := []string{ + `/bin/sh`, "-c", fmt.Sprintf(`curl -s -o /dev/null -w %%{http_code} %s`, url), + } + trafficErr := wait.PollUntilContextTimeout(context.TODO(), interval, timeout, true, func(ctx context.Context) (bool, error) { + stdOut, _, err := testData.runCommandFromPod(ns, podname, containername, cmd) + if err != nil { + return false, nil + } + statusCode := strings.Trim(stdOut, `"`) + if statusCode != "200" { + log.Info("Failed to access ip", "ip", ip, "statusCode", statusCode) + return false, nil + } + return true, nil + }) + return trafficErr +} diff --git a/test/e2e/manifest/testSecurityPolicy/allow-client-a-via-pod-selector-with-match-expressions.yaml b/test/e2e/manifest/testSecurityPolicy/allow-client-a-via-pod-selector-with-match-expressions.yaml index 71b36ca6b..ed12f6cc6 100644 --- a/test/e2e/manifest/testSecurityPolicy/allow-client-a-via-pod-selector-with-match-expressions.yaml +++ b/test/e2e/manifest/testSecurityPolicy/allow-client-a-via-pod-selector-with-match-expressions.yaml @@ -10,10 +10,7 @@ spec: restartPolicy: Never containers: - name: client-a - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/humanux/\ - http_https_echo:latest" - command: ["/bin/bash"] - args: ["-c", "while true; do echo hello, world!; sleep 60; done"] + image: "netfvt-docker-local.packages.vcfd.broadcom.net/humanux/http_https_echo:latest" --- apiVersion: v1 kind: Pod @@ -27,10 +24,7 @@ spec: restartPolicy: Never containers: - name: client-b - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/humanux/\ - http_https_echo:latest" - command: ["/bin/bash"] - args: ["-c", "while true; do echo hello, world!; sleep 60; done"] + image: "netfvt-docker-local.packages.vcfd.broadcom.net/humanux/http_https_echo:latest" --- apiVersion: v1 kind: Pod @@ -40,7 +34,4 @@ spec: restartPolicy: Never containers: - name: pod-a - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/humanux/\ - http_https_echo:latest" - command: ["/bin/bash"] - args: ["-c", "while true; do echo hello, world!; sleep 60; done"] \ No newline at end of file + image: "netfvt-docker-local.packages.vcfd.broadcom.net/humanux/http_https_echo:latest" \ No newline at end of file diff --git a/test/e2e/manifest/testSecurityPolicy/busybox.yaml b/test/e2e/manifest/testSecurityPolicy/busybox.yaml deleted file mode 100644 index 3986ba234..000000000 --- a/test/e2e/manifest/testSecurityPolicy/busybox.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: busybox -spec: - containers: - - name: busybox - image: wcp-docker-ci.artifactory.eng.vmware.com/test/busybox - command: - - /bin/sh - - "-c" - - "while true;do echo hello, world!; sleep 1; done" - imagePullPolicy: IfNotPresent - restartPolicy: Always diff --git a/test/e2e/manifest/testSecurityPolicy/netcat-pod.yaml b/test/e2e/manifest/testSecurityPolicy/netcat-pod.yaml deleted file mode 100644 index 9250f231b..000000000 --- a/test/e2e/manifest/testSecurityPolicy/netcat-pod.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: nc-pod - labels: - role: client -spec: - containers: - - name: nc - image: "harbor-repo.vmware.com/dockerhub-proxy-cache\ - /gophernet/netcat:latest" - command: ["nc", "-l", "-p", "1234"] \ No newline at end of file diff --git a/test/e2e/manifest/testSecurityPolicy/rule-in-policy-applied-to.yaml b/test/e2e/manifest/testSecurityPolicy/rule-in-policy-applied-to.yaml index 1f0bed60a..78aa74694 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-in-policy-applied-to.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-in-policy-applied-to.yaml @@ -9,7 +9,7 @@ metadata: spec: containers: - name: client - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/library/photon:3.0-20210507" + image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/gateway-reg-docker-vr/amd64/photon:sha256__0df184f96396bbcdb3d088880fbbba3384589c466b14b6e496492a7347781fe0" command: [ "/bin/bash" ] args: [ "-c", "while true; do echo hello, world!; sleep 60; done" ] restartPolicy: Never @@ -33,7 +33,7 @@ spec: hostname: web-deployment containers: - name: web - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/humanux/http_https_echo:latest" + image: "netfvt-docker-local.packages.vcfd.broadcom.net/humanux/http_https_echo:latest" imagePullPolicy: IfNotPresent ports: - containerPort: 80 diff --git a/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to-with-src.yaml b/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to-with-src.yaml index 166bdd10e..8c47a44e7 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to-with-src.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to-with-src.yaml @@ -35,7 +35,7 @@ spec: hostname: web-deployment containers: - name: web - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/humanux/http_https_echo:latest" + image: "netfvt-docker-local.packages.vcfd.broadcom.net/humanux/http_https_echo:latest" imagePullPolicy: IfNotPresent ports: - containerPort: 80 diff --git a/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to.yaml b/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to.yaml index c21ada47e..99e2eec90 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-in-rule-applied-to.yaml @@ -9,7 +9,7 @@ metadata: spec: containers: - name: client - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/library/photon:3.0-20210507" + image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/gateway-reg-docker-vr/amd64/photon:sha256__0df184f96396bbcdb3d088880fbbba3384589c466b14b6e496492a7347781fe0" command: [ "/bin/bash" ] args: [ "-c", "while true; do echo hello, world!; sleep 60; done" ] restartPolicy: Never @@ -33,7 +33,7 @@ spec: hostname: web-deployment containers: - name: web - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/humanux/http_https_echo:latest" + image: "netfvt-docker-local.packages.vcfd.broadcom.net/humanux/http_https_echo:latest" imagePullPolicy: IfNotPresent ports: - containerPort: 80 diff --git a/test/e2e/manifest/testSecurityPolicy/rule-out-policy-applied-to-with-dst.yaml b/test/e2e/manifest/testSecurityPolicy/rule-out-policy-applied-to-with-dst.yaml index a94e8e099..09132874b 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-out-policy-applied-to-with-dst.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-out-policy-applied-to-with-dst.yaml @@ -19,7 +19,7 @@ spec: role: db spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "3306"] name: db ports: @@ -45,7 +45,7 @@ spec: hostname: web-deployment containers: - name: web - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "1234"] imagePullPolicy: IfNotPresent --- diff --git a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port-multi.yaml b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port-multi.yaml index a91c44be5..724be9a2d 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port-multi.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port-multi.yaml @@ -24,7 +24,7 @@ spec: role: db spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "3306"] name: mysql ports: @@ -50,7 +50,7 @@ spec: role: db spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "1234"] name: mysql2 ports: @@ -76,7 +76,7 @@ spec: spec: hostname: web-deployment containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "4321"] name: web imagePullPolicy: IfNotPresent diff --git a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port.yaml b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port.yaml index f111480a8..876968f6b 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst-with-dup-port.yaml @@ -21,7 +21,7 @@ spec: role: db spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "3306"] name: mysql ports: @@ -46,7 +46,7 @@ spec: role: db spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "3306"] name: mysql2 ports: @@ -71,7 +71,7 @@ spec: spec: hostname: web-deployment containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "12345"] name: web imagePullPolicy: IfNotPresent diff --git a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst.yaml b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst.yaml index db9752d02..149f1b3ce 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-dst.yaml @@ -22,7 +22,7 @@ spec: role: db spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "3306"] name: mysql ports: @@ -49,7 +49,7 @@ spec: hostname: web-deployment containers: - name: web - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "1234"] imagePullPolicy: IfNotPresent --- diff --git a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-expression-selector.yaml b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-expression-selector.yaml index 7f27ff796..c50a0728d 100644 --- a/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-expression-selector.yaml +++ b/test/e2e/manifest/testSecurityPolicy/rule-out-rule-applied-to-with-expression-selector.yaml @@ -23,7 +23,7 @@ spec: k2: a2 spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "3306"] name: mysql ports: @@ -49,7 +49,7 @@ spec: role: db spec: containers: - - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + - image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "1234"] name: mysql2 ports: @@ -76,7 +76,7 @@ spec: hostname: web-deployment containers: - name: web - image: "harbor-repo.vmware.com/dockerhub-proxy-cache/gophernet/netcat:latest" + image: "netfvt-docker-local.packages.vcfd.broadcom.net:8080/wcp-docker-ci/gophernet/netcat:latest" command: ["nc", "-l", "-p", "4321"] imagePullPolicy: IfNotPresent --- diff --git a/test/e2e/nsx_security_policy_test.go b/test/e2e/nsx_security_policy_test.go index 129b9718f..81b9c9f65 100644 --- a/test/e2e/nsx_security_policy_test.go +++ b/test/e2e/nsx_security_policy_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -31,6 +32,12 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" ) +const ( + timeInterval = 1 * time.Second + timeOut10 = 10 * time.Second + timeOut5 = 5 * time.Second +) + func TestSecurityPolicy(t *testing.T) { t.Run("testSecurityPolicyBasicTraffic", func(t *testing.T) { testSecurityPolicyBasicTraffic(t) }) t.Run("testSecurityPolicyAddDeleteRule", func(t *testing.T) { testSecurityPolicyAddDeleteRule(t) }) @@ -43,36 +50,41 @@ func TestSecurityPolicy(t *testing.T) { func testSecurityPolicyBasicTraffic(t *testing.T) { deadlineCtx, deadlineCancel := context.WithTimeout(context.Background(), defaultTimeout) defer deadlineCancel() + ns := "test-security-policy-1" securityPolicyName := "isolate-policy-1" ruleName0 := "all_ingress_isolation" ruleName1 := "all_egress_isolation" - var err error - setupTest(t, ns) - defer teardownTest(t, ns, defaultTimeout) - // Temporarily disable traffic check - /* - // Create pods - busyboxPath, _ := filepath.Abs("./manifest/testSecurityPolicy/busybox.yaml") - _ = applyYAML(busyboxPath, ns) - netcatPath, _ := filepath.Abs("./manifest/testSecurityPolicy/netcat-pod.yaml") - _ = applyYAML(netcatPath, ns) - - busybox := "busybox" - ncPod := "nc-pod - // Wait for pods - ps, err := testData.podWaitForIPs(defaultTimeout, busybox, ns) - t.Logf("Pods are %v", ps) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", busybox) - iPs, err := testData.podWaitForIPs(defaultTimeout, ncPod, ns) - t.Logf("Pods are %v", iPs) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", ncPod) - - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) - */ + err := testData.createVCNamespace(ns) + if err != nil { + t.Fatalf("Failed to create VC namespace: %v", err) + } + defer func() { + err := testData.deleteVCNamespace(ns) + if err != nil { + t.Fatalf("Failed to delete VC namespace: %v", err) + } + }() + + // Create pods + clientPodName := "client-pod" + serverPodName := "server-pod" + _, err = testData.createPod(ns, clientPodName, containerName, podImage, corev1.ProtocolTCP, podPort) + require.NoErrorf(t, err, "Client Pod '%s/%s' should be created", ns, clientPodName) + _, err = testData.createPod(ns, serverPodName, containerName, podImage, corev1.ProtocolTCP, podPort) + require.NoErrorf(t, err, "Server Pod '%s/%s' should be created", ns, serverPodName) + _, err = testData.podWaitForIPs(resourceReadyTime, clientPodName, ns) + require.NoErrorf(t, err, "Client Pod '%s/%s' is not ready within time %s", ns, clientPodName, resourceReadyTime.String()) + iPs, err := testData.podWaitForIPs(resourceReadyTime, serverPodName, ns) + require.NoErrorf(t, err, "Server Pod '%s/%s' is not ready within time %s", ns, serverPodName, resourceReadyTime.String()) + log.Info("Server Pod in the Namespace is ready", "Namespace", ns, "Pod", serverPodName) + + // Test traffic from client Pod to server Pod + trafficErr := checkTrafficByCurl(ns, clientPodName, containerName, iPs.ipv4.String(), podPort, timeInterval, timeOut10) + require.NoError(t, trafficErr, "Basic traffic should work") + log.Info("Verified traffic from client Pod to the server Pod") + // Create security policy nsIsolationPath, _ := filepath.Abs("./manifest/testSecurityPolicy/ns-isolation-policy.yaml") require.NoError(t, applyYAML(nsIsolationPath, ns)) @@ -84,12 +96,10 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName0, true)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName1, true)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assertNotNil(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Test traffic from client Pod to server Pod + trafficErr = checkTrafficByCurl(ns, clientPodName, containerName, iPs.ipv4.String(), podPort, timeInterval, timeOut5) + require.Error(t, trafficErr, "Basic traffic should not work") + log.Info("Verified traffic from client Pod to the server Pod") // Delete security policy _ = deleteYAML(nsIsolationPath, ns) @@ -111,12 +121,10 @@ func testSecurityPolicyBasicTraffic(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName0, false)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName1, false)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, busybox, iPs, 4) - assert.NoError(t, err, "Error when running ping command from test Pod %s", busybox) - */ + // Test traffic from client Pod to server Pod + trafficErr = checkTrafficByCurl(ns, clientPodName, containerName, iPs.ipv4.String(), podPort, timeInterval, timeOut10) + require.NoError(t, trafficErr, "Basic traffic should work") + log.Info("Verified traffic from client Pod to the server Pod") } // TestSecurityPolicyAddDeleteRule verifies that when adding or deleting rule, the security policy will be updated. @@ -185,37 +193,41 @@ func testSecurityPolicyMatchExpression(t *testing.T) { securityPolicyName := "expression-policy-1" ruleName := "expression-policy-1-rule" - var err error - setupTest(t, ns) - defer teardownTest(t, ns, defaultTimeout) + err := testData.createVCNamespace(ns) + if err != nil { + t.Fatalf("Failed to create VC namespace: %v", err) + } + defer func() { + err := testData.deleteVCNamespace(ns) + if err != nil { + t.Fatalf("Failed to delete VC namespace: %v", err) + } + }() // Create pods podPath, _ := filepath.Abs("./manifest/testSecurityPolicy/allow-client-a-via-pod-selector-with-match-expressions.yaml") require.NoError(t, applyYAML(podPath, ns)) defer deleteYAML(podPath, "") - // Temporarily disable traffic check - // clientA := "client-a" - // clientB := "client-b" - // podA := "pod-a" - /* - // Wait for pods - ps, err := testData.podWaitForIPs(defaultTimeout, clientA, ns) - t.Logf("Pods are %v", ps) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientA) - psb, err := testData.podWaitForIPs(defaultTimeout, clientB, ns) - t.Logf("Pods are %v", psb) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientB) - iPs, err := testData.podWaitForIPs(defaultTimeout, podA, ns) - t.Logf("Pods are %v", iPs) - assert.NoError(t, err, "Error when waiting for IP for Pod %s", podA) - - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) - */ + clientA := "client-a" + clientB := "client-b" + podA := "pod-a" + // Wait for pods + _, err = testData.podWaitForIPs(defaultTimeout, clientA, ns) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientA) + _, err = testData.podWaitForIPs(defaultTimeout, clientB, ns) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", clientB) + iPs, err := testData.podWaitForIPs(defaultTimeout, podA, ns) + assert.NoError(t, err, "Error when waiting for IP for Pod %s", podA) + + // Test traffic from clientA to podA + trafficErr := checkTrafficByCurl(ns, clientA, clientA, iPs.ipv4.String(), podPort, timeInterval, timeOut10) + require.NoError(t, trafficErr, "TestSecurityPolicyMatchExpression traffic should work") + log.Info("Verified traffic from client Pod to PodA") + // Test traffic from clientB to podA + trafficErr = checkTrafficByCurl(ns, clientB, clientB, iPs.ipv4.String(), podPort, timeInterval, timeOut10) + require.NoError(t, trafficErr, "TestSecurityPolicyMatchExpression traffic should work") + log.Info("Verified traffic from client Pod to PodB") // Create security policy nsIsolationPath, _ := filepath.Abs("./manifest/testSecurityPolicy/match-expression.yaml") @@ -227,14 +239,14 @@ func testSecurityPolicyMatchExpression(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeSecurityPolicy, securityPolicyName, true)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName, true)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NotNilf(t, err, "Error when running ping command from Pod %s", clientB) - */ + // Test traffic from clientA to podA + trafficErr = checkTrafficByCurl(ns, clientA, clientA, iPs.ipv4.String(), podPort, timeInterval, timeOut10) + require.NoError(t, trafficErr, "TestSecurityPolicyMatchExpression traffic should work") + log.Info("Verified traffic from client Pod to PodA") + // Test traffic from clientB to podA + trafficErr = checkTrafficByCurl(ns, clientB, clientB, iPs.ipv4.String(), podPort, timeInterval, timeOut5) + require.Error(t, trafficErr, "TestSecurityPolicyMatchExpression traffic should not work") + log.Info("Verified traffic from client Pod to PodB") // Delete security policy _ = deleteYAML(nsIsolationPath, ns) @@ -256,14 +268,14 @@ func testSecurityPolicyMatchExpression(t *testing.T) { assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeSecurityPolicy, securityPolicyName, false)) assert.NoError(t, testData.waitForResourceExistOrNot(ns, common.ResourceTypeRule, ruleName, false)) - // Temporarily disable traffic check - /* - // Ping from pod - err = testData.runPingCommandFromPod(ns, clientA, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientA) - err = testData.runPingCommandFromPod(ns, clientB, iPs, 4) - assert.NoError(t, err, "Error when running ping command from Pod %s", clientB) - */ + // Test traffic from clientA to podA + trafficErr = checkTrafficByCurl(ns, clientA, clientA, iPs.ipv4.String(), podPort, timeInterval, timeOut10) + require.NoError(t, trafficErr, "TestSecurityPolicyMatchExpression traffic should work") + log.Info("Verified traffic from client Pod to PodA") + // Test traffic from clientB to podA + trafficErr = checkTrafficByCurl(ns, clientB, clientB, iPs.ipv4.String(), podPort, timeInterval, timeOut10) + require.NoError(t, trafficErr, "TestSecurityPolicyMatchExpression traffic should work") + log.Info("Verified traffic from client Pod to PodB") } // TestSecurityPolicyNamedPortWithoutPod verifies that the traffic of security policy when named port applied. @@ -307,7 +319,7 @@ func testSecurityPolicyNamedPortWithoutPod(t *testing.T) { func assureSecurityPolicyReady(t *testing.T, ns, spName string) { deadlineCtx, deadlineCancel := context.WithTimeout(context.Background(), defaultTimeout) defer deadlineCancel() - err := wait.PollUntilContextTimeout(deadlineCtx, 1*time.Second, defaultTimeout, false, func(ctx context.Context) (done bool, err error) { + err := wait.PollUntilContextTimeout(deadlineCtx, timeInterval, defaultTimeout, false, func(ctx context.Context) (done bool, err error) { resp, err := testData.crdClientset.CrdV1alpha1().SecurityPolicies(ns).Get(context.Background(), spName, v1.GetOptions{}) log.V(2).Info("Get resources", "SecurityPolicies", resp, "Namespace", ns, "Name", spName) if err != nil { diff --git a/test/e2e/precreated_vpc_test.go b/test/e2e/precreated_vpc_test.go index b45483c55..dd04f46db 100644 --- a/test/e2e/precreated_vpc_test.go +++ b/test/e2e/precreated_vpc_test.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "regexp" - "strings" "testing" "time" @@ -125,22 +124,7 @@ func TestPreCreatedVPC(t *testing.T) { log.Info("Client Pod in the Namespace is ready", "Namespace", nsName, "Service", svcName, "Pod", clientPodName) // Test traffic from client Pod to LB Service - url := fmt.Sprintf("http://%s:%d", lbIP, lbServicePort) - cmd := []string{ - `/bin/sh`, "-c", fmt.Sprintf(`curl -s -o /dev/null -w %%{http_code} %s`, url), - } - trafficErr := wait.PollUntilContextTimeout(context.TODO(), 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { - stdOut, _, err := testData.runCommandFromPod(nsName, clientPodName, containerName, cmd) - if err != nil { - return false, nil - } - statusCode := strings.Trim(stdOut, `"`) - if statusCode != "200" { - log.Info("Failed to access LoadBalancer", "statusCode", statusCode) - return false, nil - } - return true, nil - }) + trafficErr := checkTrafficByCurl(nsName, clientPodName, containerName, lbIP, lbServicePort, 5*time.Second, 30*time.Second) require.NoError(t, trafficErr, "LoadBalancer traffic should work") log.Info("Verified traffic from client Pod to the LoadBalancer Service") diff --git a/test/e2e/vclient.go b/test/e2e/vclient.go index 82f15bc0e..0f47194bd 100644 --- a/test/e2e/vclient.go +++ b/test/e2e/vclient.go @@ -151,13 +151,12 @@ func (c *vcClient) getSupervisorID() (string, error) { } func (c *vcClient) createNamespaceWithPreCreatedVPC(namespace string, vpcPath string, supervisorID string) error { - urlPath := "/api/vcenter/namespaces/instances/v2" vcNamespace := createVCNamespaceSpec(namespace, supervisorID, vpcPath) data, err := json.Marshal(vcNamespace) if err != nil { return fmt.Errorf("unable convert vcNamespace object to json bytes: %v", err) } - request, err := c.prepareRequest(http.MethodPost, urlPath, data) + request, err := c.prepareRequest(http.MethodPost, createVCNamespaceEndpoint, data) if err != nil { return fmt.Errorf("failed to parepare http request with vcNamespace data: %v", err) }