Improved common test setup

1. Move common test setup to its own package so it can be shared
between test suites.

2. Improve the logic for starting the manager by:

- Separate the build and run commands. "go run" runs in its own process
so it's more difficult to find the process of "workspace-manger" and
kill it at the end of the test suite. This change fixing a bug
where the test suite didn't kill the workspace-manager process on exit.

- Wait for the server to be ready to serve http requests. For that
an additional endpoint /health was added to the server.

- Write the workspace-manager out to a log file.

3. Additional test suite was added for testing namespace provisioning.
It currently contains a dummy test. The actual test will be added
with the logic for provisioning namespaces. It was useful to add
and additional test suite (even a dummy one) in this change for verifying
the commons test setup code.

Signed-off-by: gbenhaim <[email protected]>

cmd/main.go

@@ -263,5 +263,9 @@ func main() {
 		return echo.NewHTTPError(http.StatusNotFound)
 	})
 
+	e.GET("/health", func(c echo.Context) error {
+		return c.NoContent(http.StatusOK)
+	})
+
 	e.Logger.Fatal(e.Start(":5000"))
 }

cmd/main_test.go

@@ -7,30 +7,31 @@ import (
 	"net/http"
 	"os/exec"
 	"strings"
-	"time"
 
 	"github.com/labstack/echo/v4"
 	k8sapi "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/client-go/kubernetes"
 
 	"context"
 	"net/http/httptest"
-	"os"
 	"testing"
 
 	crt "github.com/codeready-toolchain/api/api/v1alpha1"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
+	"github.com/konflux-ci/workspace-manager/pkg/test/utils"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 )
 
 type HTTPResponse struct {
@@ -250,46 +251,17 @@ var _ = DescribeTable("Specific workspace endpoint", func(endpoint string, heade
 		`{"message":"Not Found"}`),
 )
 
-func CreateKubeconfigFileForRestConfig(restConfig rest.Config) string {
-	clusters := make(map[string]*clientcmdapi.Cluster)
-	clusters["default-cluster"] = &clientcmdapi.Cluster{
-		Server:                   restConfig.Host,
-		CertificateAuthorityData: restConfig.CAData,
-	}
-	contexts := make(map[string]*clientcmdapi.Context)
-	contexts["default-context"] = &clientcmdapi.Context{
-		Cluster:  "default-cluster",
-		AuthInfo: "default-user",
-	}
-	authinfos := make(map[string]*clientcmdapi.AuthInfo)
-	authinfos["default-user"] = &clientcmdapi.AuthInfo{
-		ClientCertificateData: restConfig.CertData,
-		ClientKeyData:         restConfig.KeyData,
-	}
-	clientConfig := clientcmdapi.Config{
-		Kind:           "Config",
-		APIVersion:     "v1",
-		Clusters:       clusters,
-		Contexts:       contexts,
-		CurrentContext: "default-context",
-		AuthInfos:      authinfos,
-	}
-	kubeConfigFile, _ := os.CreateTemp("", "kubeconfig")
-	_ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name())
-	return kubeConfigFile.Name()
-}
-
 var serverProcess *exec.Cmd
+var serverCancelFunc context.CancelFunc
 
 var _ = BeforeSuite(func() {
-	testEnv = &envtest.Environment{}
-	cfg, err := testEnv.Start()
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the envtest environment during test setup: %v", err))
-	kubeconfigPath := CreateKubeconfigFileForRestConfig(*cfg)
-	os.Setenv("KUBECONFIG", kubeconfigPath)
-	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the client during test setup: %v", err))
-	Expect(k8sClient).NotTo(BeNil())
+	schema := runtime.NewScheme()
+	utilruntime.Must(clientgoscheme.AddToScheme(schema))
+	testEnv = &envtest.Environment{BinaryAssetsDirectory: "../bin/k8s/1.29.0-linux-amd64/"}
+	k8sClient = utils.StartTestEnv(schema, testEnv)
+
+	serverProcess, serverCancelFunc = utils.CreateWorkspaceManagerServer("main.go", nil, "")
+	utils.WaitForWorkspaceManagerServerToServe()
 
 	user1 := "[email protected]"
 	user2 := "[email protected]"
@@ -303,18 +275,11 @@ var _ = BeforeSuite(func() {
 	createRoleBinding(k8sClient, "namespace-access-user-binding", "test-tenant", user1, "namespace-access")
 	createRoleBinding(k8sClient, "namespace-access-user-binding-2", "test-tenant", user2, "namespace-access")
 	createRoleBinding(k8sClient, "namespace-access-user-binding-3", "test-tenant-2", user2, "namespace-access-2")
-	serverProcess = exec.Command("go", "run", "main.go")
-	err = serverProcess.Start()
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error starting the server during test setup: %v", err))
-	time.Sleep(5 * time.Second)
 })
 
 var _ = AfterSuite(func() {
-	Expect(os.Unsetenv("KUBECONFIG")).To(Succeed())
-	if serverProcess != nil && serverProcess.Process != nil {
-		err := serverProcess.Process.Kill()
-		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error killing the server during test teardown: %v", err))
-	}
+	utils.StopWorkspaceManagerServer(serverProcess, serverCancelFunc)
+	utils.StopEnvTest(testEnv)
 })
 
 var _ = DescribeTable("TestRunAccessCheck", func(user string, namespace string, resource string, verb string, expectedResult bool) {

pkg/test/provision-test/provision_test.go

@@ -0,0 +1,68 @@
+package provision_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os/exec"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/konflux-ci/workspace-manager/pkg/test/utils"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+)
+
+func TestProvision(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Provision namespace Suite")
+}
+
+var k8sClient client.Client
+var testEnv *envtest.Environment
+var serverProcess *exec.Cmd
+var serverCancelFunc context.CancelFunc
+
+var _ = BeforeSuite(func() {
+	schema := runtime.NewScheme()
+	utilruntime.Must(clientgoscheme.AddToScheme(schema))
+	testEnv = &envtest.Environment{BinaryAssetsDirectory: "../../../bin/k8s/1.29.0-linux-amd64/"}
+	k8sClient = utils.StartTestEnv(schema, testEnv)
+	serverProcess, serverCancelFunc = utils.CreateWorkspaceManagerServer("../../../cmd/main.go", nil, "")
+	utils.WaitForWorkspaceManagerServerToServe()
+})
+
+var _ = AfterSuite(func() {
+	utils.StopWorkspaceManagerServer(serverProcess, serverCancelFunc)
+	utils.StopEnvTest(testEnv)
+})
+
+var _ = Describe("simple test", func() {
+	endpoint := "http://localhost:5000/api/v1/signup"
+	httpClient := &http.Client{}
+
+	Context("simple test context", func() {
+		It("simple spec", func() {
+			request, err := http.NewRequest("GET", endpoint, nil)
+			Expect(err).NotTo(HaveOccurred())
+			Eventually(
+				func() (int, error) {
+					response, err := httpClient.Do(request)
+					if err != nil {
+						fmt.Println(err.Error())
+						return 0, err
+					}
+					return response.StatusCode, nil
+
+				},
+				10,
+				1,
+			).Should(Equal(http.StatusOK))
+		})
+	})
+})

pkg/test/utils/utils.go

@@ -0,0 +1,166 @@
+package utils
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+)
+
+// Start the envtest environment
+func StartTestEnv(scheme *runtime.Scheme, testEnv *envtest.Environment) client.Client {
+	cfg, err := testEnv.Start()
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the envtest environment during test setup: %v", err))
+	kubeconfigPath := CreateKubeconfigFileForRestConfig(*cfg)
+	os.Setenv("KUBECONFIG", kubeconfigPath)
+	k8sClient, err := client.New(cfg, client.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the client during test setup: %v", err))
+	Expect(k8sClient).NotTo(BeNil())
+
+	return k8sClient
+}
+
+// Stop the envtest environment
+func StopEnvTest(envTest *envtest.Environment) {
+	if envTest != nil {
+		err := envTest.Stop()
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to stop envTest: %v", err))
+	}
+}
+
+// Create a Kubeconfig from the given rest config.
+func CreateKubeconfigFileForRestConfig(restConfig rest.Config) string {
+	clusters := make(map[string]*clientcmdapi.Cluster)
+	clusters["default-cluster"] = &clientcmdapi.Cluster{
+		Server:                   restConfig.Host,
+		CertificateAuthorityData: restConfig.CAData,
+	}
+	contexts := make(map[string]*clientcmdapi.Context)
+	contexts["default-context"] = &clientcmdapi.Context{
+		Cluster:  "default-cluster",
+		AuthInfo: "default-user",
+	}
+	authinfos := make(map[string]*clientcmdapi.AuthInfo)
+	authinfos["default-user"] = &clientcmdapi.AuthInfo{
+		ClientCertificateData: restConfig.CertData,
+		ClientKeyData:         restConfig.KeyData,
+	}
+	clientConfig := clientcmdapi.Config{
+		Kind:           "Config",
+		APIVersion:     "v1",
+		Clusters:       clusters,
+		Contexts:       contexts,
+		CurrentContext: "default-context",
+		AuthInfos:      authinfos,
+	}
+	kubeConfigFile, _ := os.CreateTemp("", "kubeconfig")
+	_ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name())
+	return kubeConfigFile.Name()
+}
+
+// Build the workspace-manager binary.
+// mainPath is the path to the main module.
+func BuildWorkspaceManager(mainPath string) string {
+	out := os.TempDir()
+	binPath := filepath.Join(out, "workspace-manager", "manager")
+	buildCmd := exec.Command("go", "build", "-o", binPath, mainPath)
+	buildLog, err := buildCmd.CombinedOutput()
+	Expect(err).NotTo(
+		HaveOccurred(),
+		"Failed to build the manager, %s\nBuild log: %s",
+		err,
+		buildLog,
+	)
+
+	return binPath
+}
+
+// Start workspace manager in the background. Return its backing Cmd and
+// a function that can be used to kill its process.
+// binPath is the path to the workspace-manager binary
+// env is an array for specifying environment variables to be declared in the workspace-manager process.
+// logFile is a file that will be used for storing workspace-manager stdout and stderr.
+func StartWorkspaceManagerServer(binPath string, env []string, logFile *os.File) (*exec.Cmd, context.CancelFunc) {
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	serverCmd := exec.CommandContext(ctx, binPath)
+	serverCmd.Env = append(os.Environ(), env...)
+	serverCmd.Stderr = logFile
+	serverCmd.Stdout = logFile
+	err := serverCmd.Start()
+	Expect(err).NotTo(
+		HaveOccurred(),
+		"Failed to start the manager, %s",
+		err,
+	)
+
+	return serverCmd, cancelFunc
+}
+
+// Create a file in a temporary directory for storing workspace-manager output.
+// The path to the log file will be printed so it can be seen in the output.
+// of the test suite.
+// dir is a directory for storing the file with the workspace-manager output.
+// If empty the default temporary directory will be used.
+func CreateLogFile(dir string) *os.File {
+	tmpdir, err := os.MkdirTemp(dir, "workspace-manager")
+	Expect(err).NotTo(HaveOccurred(), "Failed to create tempdir for the logs")
+	logFile, err := os.Create(filepath.Join(tmpdir, "workspace-manager.log"))
+	Expect(err).NotTo(HaveOccurred(), "Failed to create file for the workspace-manager log")
+	fmt.Printf("workspace-manager logs will be written to: %s\n", logFile.Name())
+
+	return logFile
+}
+
+// Wait for workspace-manager to start serving http requests
+func WaitForWorkspaceManagerServerToServe() {
+	endpoint := "http://localhost:5000/health"
+	httpClient := &http.Client{}
+	request, err := http.NewRequest("GET", endpoint, nil)
+	Expect(err).NotTo(HaveOccurred())
+	fmt.Println("Waiting for workspace-manager to start. You may see some errors printed to the log.")
+	Eventually(
+		func() (int, error) {
+			response, err := httpClient.Do(request)
+			if err != nil {
+				fmt.Println(err.Error())
+				return 0, err
+			}
+			return response.StatusCode, nil
+
+		},
+		10,
+		1,
+	).Should(Equal(http.StatusOK), "Wait for Workspace Manager server to start")
+}
+
+// Build and start workspace-manager
+// mainPath is the path to the main module
+// env is an array for specifying environment variables to be declared in the workspace-manager process.
+// logsDir is a directory for storing the file with the workspace-manager output.
+// If empty the default temporary directory will be used.
+func CreateWorkspaceManagerServer(mainPath string, env []string, logsDir string) (*exec.Cmd, context.CancelFunc) {
+	return StartWorkspaceManagerServer(
+		BuildWorkspaceManager(mainPath),
+		env,
+		CreateLogFile(logsDir),
+	)
+}
+
+// Stop the workspace-manager process and wait for it to be stopped
+func StopWorkspaceManagerServer(cmd *exec.Cmd, serverCancelFunc context.CancelFunc) {
+	if cmd != nil {
+		serverCancelFunc()
+		err := cmd.Wait()
+		Expect(err).Should(BeAssignableToTypeOf(&exec.ExitError{}))
+	}
+}