From 74b2df5d2fa0305a348578ce9b270690c91c0707 Mon Sep 17 00:00:00 2001 From: gbenhaim Date: Tue, 27 Aug 2024 16:31:45 +0300 Subject: [PATCH] Improved common test setup 1. Move common test setup to its own package so it can be shared between test suites. 2. Improve the logic for starting the manager by: - Separate the build and run commands. "go run" runs in its own process so it's more difficult to find the process of "workspace-manger" and kill it at the end of the test suite. This change fixing a bug where the test suite didn't kill the workspace-manager process on exit. - Wait for the server to be ready to serve http requests. For that an additional endpoint /health was added to the server. - Write the workspace-manager out to a log file. 3. Additional test suite was added for testing namespace provisioning. It currently contains a dummy test. The actual test will be added with the logic for provisioning namespaces. It was useful to add and additional test suite (even a dummy one) in this change for verifying the commons test setup code. Signed-off-by: gbenhaim --- cmd/main.go | 4 + cmd/main_test.go | 67 +++------ pkg/test/provision-test/provision_test.go | 68 +++++++++ pkg/test/utils/utils.go | 166 ++++++++++++++++++++++ 4 files changed, 254 insertions(+), 51 deletions(-) create mode 100644 pkg/test/provision-test/provision_test.go create mode 100644 pkg/test/utils/utils.go diff --git a/cmd/main.go b/cmd/main.go index f6bfc09..25333c6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -263,5 +263,9 @@ func main() { return echo.NewHTTPError(http.StatusNotFound) }) + e.GET("/health", func(c echo.Context) error { + return c.NoContent(http.StatusOK) + }) + e.Logger.Fatal(e.Start(":5000")) } diff --git a/cmd/main_test.go b/cmd/main_test.go index 8481d3c..940fd3e 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -7,30 +7,31 @@ import ( "net/http" "os/exec" "strings" - "time" "github.com/labstack/echo/v4" k8sapi "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/kubernetes" "context" "net/http/httptest" - "os" "testing" crt "github.com/codeready-toolchain/api/api/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/envtest" + + "github.com/konflux-ci/workspace-manager/pkg/test/utils" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" ) type HTTPResponse struct { @@ -250,46 +251,17 @@ var _ = DescribeTable("Specific workspace endpoint", func(endpoint string, heade `{"message":"Not Found"}`), ) -func CreateKubeconfigFileForRestConfig(restConfig rest.Config) string { - clusters := make(map[string]*clientcmdapi.Cluster) - clusters["default-cluster"] = &clientcmdapi.Cluster{ - Server: restConfig.Host, - CertificateAuthorityData: restConfig.CAData, - } - contexts := make(map[string]*clientcmdapi.Context) - contexts["default-context"] = &clientcmdapi.Context{ - Cluster: "default-cluster", - AuthInfo: "default-user", - } - authinfos := make(map[string]*clientcmdapi.AuthInfo) - authinfos["default-user"] = &clientcmdapi.AuthInfo{ - ClientCertificateData: restConfig.CertData, - ClientKeyData: restConfig.KeyData, - } - clientConfig := clientcmdapi.Config{ - Kind: "Config", - APIVersion: "v1", - Clusters: clusters, - Contexts: contexts, - CurrentContext: "default-context", - AuthInfos: authinfos, - } - kubeConfigFile, _ := os.CreateTemp("", "kubeconfig") - _ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name()) - return kubeConfigFile.Name() -} - var serverProcess *exec.Cmd +var serverCancelFunc context.CancelFunc var _ = BeforeSuite(func() { - testEnv = &envtest.Environment{} - cfg, err := testEnv.Start() - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the envtest environment during test setup: %v", err)) - kubeconfigPath := CreateKubeconfigFileForRestConfig(*cfg) - os.Setenv("KUBECONFIG", kubeconfigPath) - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the client during test setup: %v", err)) - Expect(k8sClient).NotTo(BeNil()) + schema := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(schema)) + testEnv = &envtest.Environment{BinaryAssetsDirectory: "../bin/k8s/1.29.0-linux-amd64/"} + k8sClient = utils.StartTestEnv(schema, testEnv) + + serverProcess, serverCancelFunc = utils.CreateWorkspaceManagerServer("main.go", nil, "") + utils.WaitForWorkspaceManagerServerToServe() user1 := "user1@konflux.dev" user2 := "user2@konflux.dev" @@ -303,18 +275,11 @@ var _ = BeforeSuite(func() { createRoleBinding(k8sClient, "namespace-access-user-binding", "test-tenant", user1, "namespace-access") createRoleBinding(k8sClient, "namespace-access-user-binding-2", "test-tenant", user2, "namespace-access") createRoleBinding(k8sClient, "namespace-access-user-binding-3", "test-tenant-2", user2, "namespace-access-2") - serverProcess = exec.Command("go", "run", "main.go") - err = serverProcess.Start() - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error starting the server during test setup: %v", err)) - time.Sleep(5 * time.Second) }) var _ = AfterSuite(func() { - Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) - if serverProcess != nil && serverProcess.Process != nil { - err := serverProcess.Process.Kill() - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error killing the server during test teardown: %v", err)) - } + utils.StopWorkspaceManagerServer(serverProcess, serverCancelFunc) + utils.StopEnvTest(testEnv) }) var _ = DescribeTable("TestRunAccessCheck", func(user string, namespace string, resource string, verb string, expectedResult bool) { diff --git a/pkg/test/provision-test/provision_test.go b/pkg/test/provision-test/provision_test.go new file mode 100644 index 0000000..ac09d5f --- /dev/null +++ b/pkg/test/provision-test/provision_test.go @@ -0,0 +1,68 @@ +package provision_test + +import ( + "context" + "fmt" + "net/http" + "os/exec" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/konflux-ci/workspace-manager/pkg/test/utils" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func TestProvision(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Provision namespace Suite") +} + +var k8sClient client.Client +var testEnv *envtest.Environment +var serverProcess *exec.Cmd +var serverCancelFunc context.CancelFunc + +var _ = BeforeSuite(func() { + schema := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(schema)) + testEnv = &envtest.Environment{BinaryAssetsDirectory: "../../../bin/k8s/1.29.0-linux-amd64/"} + k8sClient = utils.StartTestEnv(schema, testEnv) + serverProcess, serverCancelFunc = utils.CreateWorkspaceManagerServer("../../../cmd/main.go", nil, "") + utils.WaitForWorkspaceManagerServerToServe() +}) + +var _ = AfterSuite(func() { + utils.StopWorkspaceManagerServer(serverProcess, serverCancelFunc) + utils.StopEnvTest(testEnv) +}) + +var _ = Describe("simple test", func() { + endpoint := "http://localhost:5000/api/v1/signup" + httpClient := &http.Client{} + + Context("simple test context", func() { + It("simple spec", func() { + request, err := http.NewRequest("GET", endpoint, nil) + Expect(err).NotTo(HaveOccurred()) + Eventually( + func() (int, error) { + response, err := httpClient.Do(request) + if err != nil { + fmt.Println(err.Error()) + return 0, err + } + return response.StatusCode, nil + + }, + 10, + 1, + ).Should(Equal(http.StatusOK)) + }) + }) +}) diff --git a/pkg/test/utils/utils.go b/pkg/test/utils/utils.go new file mode 100644 index 0000000..9027cb6 --- /dev/null +++ b/pkg/test/utils/utils.go @@ -0,0 +1,166 @@ +package utils + +import ( + "context" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// Start the envtest environment +func StartTestEnv(scheme *runtime.Scheme, testEnv *envtest.Environment) client.Client { + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the envtest environment during test setup: %v", err)) + kubeconfigPath := CreateKubeconfigFileForRestConfig(*cfg) + os.Setenv("KUBECONFIG", kubeconfigPath) + k8sClient, err := client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error creating the client during test setup: %v", err)) + Expect(k8sClient).NotTo(BeNil()) + + return k8sClient +} + +// Stop the envtest environment +func StopEnvTest(envTest *envtest.Environment) { + if envTest != nil { + err := envTest.Stop() + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to stop envTest: %v", err)) + } +} + +// Create a Kubeconfig from the given rest config. +func CreateKubeconfigFileForRestConfig(restConfig rest.Config) string { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + AuthInfo: "default-user", + } + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos["default-user"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authinfos, + } + kubeConfigFile, _ := os.CreateTemp("", "kubeconfig") + _ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name()) + return kubeConfigFile.Name() +} + +// Build the workspace-manager binary. +// mainPath is the path to the main module. +func BuildWorkspaceManager(mainPath string) string { + out := os.TempDir() + binPath := filepath.Join(out, "workspace-manager", "manager") + buildCmd := exec.Command("go", "build", "-o", binPath, mainPath) + buildLog, err := buildCmd.CombinedOutput() + Expect(err).NotTo( + HaveOccurred(), + "Failed to build the manager, %s\nBuild log: %s", + err, + buildLog, + ) + + return binPath +} + +// Start workspace manager in the background. Return its backing Cmd and +// a function that can be used to kill its process. +// binPath is the path to the workspace-manager binary +// env is an array for specifying environment variables to be declared in the workspace-manager process. +// logFile is a file that will be used for storing workspace-manager stdout and stderr. +func StartWorkspaceManagerServer(binPath string, env []string, logFile *os.File) (*exec.Cmd, context.CancelFunc) { + ctx, cancelFunc := context.WithCancel(context.Background()) + serverCmd := exec.CommandContext(ctx, binPath) + serverCmd.Env = append(os.Environ(), env...) + serverCmd.Stderr = logFile + serverCmd.Stdout = logFile + err := serverCmd.Start() + Expect(err).NotTo( + HaveOccurred(), + "Failed to start the manager, %s", + err, + ) + + return serverCmd, cancelFunc +} + +// Create a file in a temporary directory for storing workspace-manager output. +// The path to the log file will be printed so it can be seen in the output. +// of the test suite. +// dir is a directory for storing the file with the workspace-manager output. +// If empty the default temporary directory will be used. +func CreateLogFile(dir string) *os.File { + tmpdir, err := os.MkdirTemp(dir, "workspace-manager") + Expect(err).NotTo(HaveOccurred(), "Failed to create tempdir for the logs") + logFile, err := os.Create(filepath.Join(tmpdir, "workspace-manager.log")) + Expect(err).NotTo(HaveOccurred(), "Failed to create file for the workspace-manager log") + fmt.Printf("workspace-manager logs will be written to: %s\n", logFile.Name()) + + return logFile +} + +// Wait for workspace-manager to start serving http requests +func WaitForWorkspaceManagerServerToServe() { + endpoint := "http://localhost:5000/health" + httpClient := &http.Client{} + request, err := http.NewRequest("GET", endpoint, nil) + Expect(err).NotTo(HaveOccurred()) + fmt.Println("Waiting for workspace-manager to start. You may see some errors printed to the log.") + Eventually( + func() (int, error) { + response, err := httpClient.Do(request) + if err != nil { + fmt.Println(err.Error()) + return 0, err + } + return response.StatusCode, nil + + }, + 10, + 1, + ).Should(Equal(http.StatusOK), "Wait for Workspace Manager server to start") +} + +// Build and start workspace-manager +// mainPath is the path to the main module +// env is an array for specifying environment variables to be declared in the workspace-manager process. +// logsDir is a directory for storing the file with the workspace-manager output. +// If empty the default temporary directory will be used. +func CreateWorkspaceManagerServer(mainPath string, env []string, logsDir string) (*exec.Cmd, context.CancelFunc) { + return StartWorkspaceManagerServer( + BuildWorkspaceManager(mainPath), + env, + CreateLogFile(logsDir), + ) +} + +// Stop the workspace-manager process and wait for it to be stopped +func StopWorkspaceManagerServer(cmd *exec.Cmd, serverCancelFunc context.CancelFunc) { + if cmd != nil { + serverCancelFunc() + err := cmd.Wait() + Expect(err).Should(BeAssignableToTypeOf(&exec.ExitError{})) + } +}