From 4d554eeaefede406848aa9219c6fd5e5a73c05b7 Mon Sep 17 00:00:00 2001
From: Prithak S <prithak342@gmail.com>
Date: Thu, 11 Jul 2024 17:38:40 +0100
Subject: [PATCH] fix: flux kyverno tests & return stderr while reading
 kubeconfig via kwokctl

---
 .../flux/kyverno/main_test.go                 | 48 ++++++++++++++++++-
 support/kwok/kwok.go                          | 15 +++---
 2 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/examples/third_party_integration/flux/kyverno/main_test.go b/examples/third_party_integration/flux/kyverno/main_test.go
index 86bfcaeb..fe292792 100644
--- a/examples/third_party_integration/flux/kyverno/main_test.go
+++ b/examples/third_party_integration/flux/kyverno/main_test.go
@@ -17,12 +17,21 @@ limitations under the License.
 package kyverno
 
 import (
+	"context"
+	"fmt"
 	"os"
 	"testing"
+	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
 	"sigs.k8s.io/e2e-framework/pkg/env"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+	"sigs.k8s.io/e2e-framework/support/kind"
 	"sigs.k8s.io/e2e-framework/third_party/flux"
 )
 
@@ -41,12 +50,15 @@ func TestMain(m *testing.M) {
 	hrNameKyverno := "kyverno"
 	hrNamePolicies := "kyverno-policies"
 	testEnv.Setup(
-		envfuncs.CreateKindCluster(kindClusterName),
+		envfuncs.CreateCluster(kind.NewProvider(), kindClusterName),
 		envfuncs.CreateNamespace(namespace),
 		flux.InstallFlux(),
 		flux.CreateHelmRepository(helmRepoName, "https://kyverno.github.io/kyverno/"),
 		flux.CreateHelmRelease(hrNameKyverno, "HelmRepository/"+helmRepoName, "kyverno", flux.WithArgs("--target-namespace", "kyverno", "--create-target-namespace")),
 		flux.CreateHelmRelease(hrNamePolicies, "HelmRepository/"+helmRepoName, "kyverno-policies", flux.WithArgs("--target-namespace", "kyverno", "--values", "kyverno-values.yaml")),
+		func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+			return ctx, waitForKyvernoDeployments(cfg)
+		},
 	)
 
 	testEnv.Finish(
@@ -55,7 +67,39 @@ func TestMain(m *testing.M) {
 		flux.DeleteHelmRepo(helmRepoName),
 		flux.UninstallFlux(),
 		envfuncs.DeleteNamespace(namespace),
-		envfuncs.DestroyKindCluster(kindClusterName),
+		envfuncs.DestroyCluster(kindClusterName),
 	)
 	os.Exit(testEnv.Run(m))
 }
+
+// waitForKyvernoDeployments - waits for all the deployments related to kyverno to be available
+func waitForKyvernoDeployments(c *envconf.Config) error {
+	client, err := c.NewClient()
+	if err != nil {
+		return fmt.Errorf("failed to create client: %w", err)
+	}
+
+	kyvernoDeployments := []string{
+		"kyverno-admission-controller",
+		"kyverno-background-controller",
+		"kyverno-cleanup-controller",
+		"kyverno-reports-controller",
+	}
+
+	for _, deploymentName := range kyvernoDeployments {
+		deployment := &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      deploymentName,
+				Namespace: "kyverno",
+			},
+		}
+
+		err := wait.For(conditions.New(client.Resources()).DeploymentConditionMatch(deployment, appsv1.DeploymentAvailable, corev1.ConditionTrue),
+			wait.WithTimeout(time.Minute*2), wait.WithInterval(time.Second*5))
+		if err != nil {
+			return fmt.Errorf("deployment %s did not become available: %w", deploymentName, err)
+		}
+	}
+
+	return nil
+}
diff --git a/support/kwok/kwok.go b/support/kwok/kwok.go
index de41ef0b..7ec07ada 100644
--- a/support/kwok/kwok.go
+++ b/support/kwok/kwok.go
@@ -88,14 +88,11 @@ func (k *Cluster) clusterExists(name string) (string, bool) {
 func (k *Cluster) getKubeconfig() (string, error) {
 	kubecfg := fmt.Sprintf("%s-kubecfg", k.name)
 
-	p := utils.RunCommand(fmt.Sprintf(`%s get kubeconfig --name %s`, k.path, k.name))
-	if p.Err() != nil {
-		return "", fmt.Errorf("kwokctl get kubeconfig: %w", p.Err())
-	}
-
-	var stdout bytes.Buffer
-	if _, err := stdout.ReadFrom(p.Out()); err != nil {
-		return "", fmt.Errorf("kwokctl kubeconfig stdout bytes: %w", err)
+	var stdout, stderr bytes.Buffer
+	err := utils.RunCommandWithSeperatedOutput(fmt.Sprintf(`%s get kubeconfig --name %s`,
+		k.path, k.name), &stdout, &stderr)
+	if err != nil {
+		return "", fmt.Errorf("kwokctl get kubeconfig: stderr: %s: %w", stderr.String(), err)
 	}
 
 	file, err := os.CreateTemp("", fmt.Sprintf("kwok-cluster-%s", kubecfg))
@@ -106,7 +103,7 @@ func (k *Cluster) getKubeconfig() (string, error) {
 
 	k.kubecfgFile = file.Name()
 
-	if n, err := io.Copy(file, &stdout); n == 0 || err != nil {
+	if n, err := io.WriteString(file, stdout.String()); n == 0 || err != nil {
 		return "", fmt.Errorf("kwok kubecfg file: bytes copied: %d: %w]", n, err)
 	}