From c2620c4b946ed5a3bdd0632cb46f86122a6635fb Mon Sep 17 00:00:00 2001 From: Kautilya Tripathi Date: Thu, 14 Nov 2024 13:55:49 +0530 Subject: [PATCH] backend: Fix reloading of clusters in kubeconfig When a context was being added and removed from KUBECONFIG file, it was still visible in UI. This removes the cluster from store and returns correct list of clusters. Fixes #2541 Signed-off-by: Kautilya Tripathi --- backend/pkg/kubeconfig/file_test.go | 20 +++++ backend/pkg/kubeconfig/watcher.go | 55 +++++++++++- backend/pkg/kubeconfig/watcher_test.go | 117 +++++++++++++++---------- 3 files changed, 144 insertions(+), 48 deletions(-) diff --git a/backend/pkg/kubeconfig/file_test.go b/backend/pkg/kubeconfig/file_test.go index 2e94bf2cc7..65f0aa74a9 100644 --- a/backend/pkg/kubeconfig/file_test.go +++ b/backend/pkg/kubeconfig/file_test.go @@ -10,6 +10,26 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +const clusterConf = `apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: dGVzdA== + server: https://kubernetes.docker.internal:6443 + name: random-cluster-4 +contexts: +- context: + cluster: random-cluster-4 + user: random-cluster-4 + name: random-cluster-4 +current-context: random-cluster-4 +kind: Config +preferences: {} +users: +- name: random-cluster-4 + user: + client-certificate-data: dGVzdA== + client-key-data: dGVzdA==` + func TestWriteToFile(t *testing.T) { // create kubeconfig3 file that doesn't exist conf, err := clientcmd.Load([]byte(clusterConf)) diff --git a/backend/pkg/kubeconfig/watcher.go b/backend/pkg/kubeconfig/watcher.go index 8f96a797c1..24f413638a 100644 --- a/backend/pkg/kubeconfig/watcher.go +++ b/backend/pkg/kubeconfig/watcher.go @@ -1,6 +1,7 @@ package kubeconfig import ( + "fmt" "os" "path/filepath" "time" @@ -48,14 +49,13 @@ func LoadAndWatchFiles(kubeConfigStore ContextStore, paths string, source int) { case event := <-watcher.Events: triggers := []fsnotify.Op{fsnotify.Create, fsnotify.Write, fsnotify.Remove, fsnotify.Rename} for _, trigger := range triggers { - trigger := trigger if event.Op.Has(trigger) { logger.Log(logger.LevelInfo, map[string]string{"event": event.Name}, nil, "watcher: kubeconfig file changed, reloading contexts") - err := LoadAndStoreKubeConfigs(kubeConfigStore, paths, source) + err := syncContexts(kubeConfigStore, paths, source) if err != nil { - logger.Log(logger.LevelError, nil, err, "watcher: error loading kubeconfig files") + logger.Log(logger.LevelError, nil, err, "watcher: error synchronizing contexts") } } } @@ -106,3 +106,52 @@ func addFilesToWatcher(watcher *fsnotify.Watcher, paths []string) { } } } + +// syncContexts synchronizes the contexts in the store with the ones in the kubeconfig files. +func syncContexts(kubeConfigStore ContextStore, paths string, source int) error { + // First read all kubeconfig files to get new contexts + newContexts, _, err := LoadContextsFromMultipleFiles(paths, source) + if err != nil { + return fmt.Errorf("error reading kubeconfig files: %v", err) + } + + // Get existing contexts from store + existingContexts, err := kubeConfigStore.GetContexts() + if err != nil { + return fmt.Errorf("error getting existing contexts: %v", err) + } + + // Find and remove contexts that no longer exist in the kubeconfig + // but only for contexts that came from KubeConfig source + for _, existingCtx := range existingContexts { + // Skip contexts from other sources + if existingCtx.Source != KubeConfig { + continue + } + + found := false + + for _, newCtx := range newContexts { + if existingCtx.Name == newCtx.Name { + found = true + + break + } + } + + if !found { + err := kubeConfigStore.RemoveContext(existingCtx.Name) + if err != nil { + logger.Log(logger.LevelError, nil, err, "error removing context") + } + } + } + + // Now load and store the new configurations + err = LoadAndStoreKubeConfigs(kubeConfigStore, paths, source) + if err != nil { + return fmt.Errorf("error loading kubeconfig files: %v", err) + } + + return nil +} diff --git a/backend/pkg/kubeconfig/watcher_test.go b/backend/pkg/kubeconfig/watcher_test.go index 526a6ec0f8..168e0a57a7 100644 --- a/backend/pkg/kubeconfig/watcher_test.go +++ b/backend/pkg/kubeconfig/watcher_test.go @@ -1,7 +1,6 @@ package kubeconfig_test import ( - "os" "runtime" "strings" "testing" @@ -10,30 +9,12 @@ import ( "github.com/headlamp-k8s/headlamp/backend/pkg/kubeconfig" "github.com/stretchr/testify/require" "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) -const clusterConf = `apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: dGVzdA== - server: https://kubernetes.docker.internal:6443 - name: random-cluster-4 -contexts: -- context: - cluster: random-cluster-4 - user: random-cluster-4 - name: random-cluster-4 -current-context: random-cluster-4 -kind: Config -preferences: {} -users: -- name: random-cluster-4 - user: - client-certificate-data: dGVzdA== - client-key-data: dGVzdA==` - +//nolint:funlen func TestWatchAndLoadFiles(t *testing.T) { - paths := []string{"./test_data/kubeconfig1", "./test_data/kubeconfig2", "./test_data/kubeconfig3"} + paths := []string{"./test_data/kubeconfig1", "./test_data/kubeconfig2"} var path string if runtime.GOOS == "windows" { @@ -46,37 +27,83 @@ func TestWatchAndLoadFiles(t *testing.T) { go kubeconfig.LoadAndWatchFiles(kubeConfigStore, path, kubeconfig.KubeConfig) - // SLeep so the config file has a different time stamp. - time.Sleep(5 * time.Second) + // Test adding a context + t.Run("Add context", func(t *testing.T) { + // Sleep to ensure watcher is ready + time.Sleep(2 * time.Second) + + // Read existing config + config, err := clientcmd.LoadFromFile("./test_data/kubeconfig1") + require.NoError(t, err) - // create kubeconfig3 file that doesn't exist - conf, err := clientcmd.Load([]byte(clusterConf)) - require.NoError(t, err) - require.NotNil(t, conf) + // Add new context + config.Contexts["random-cluster-4"] = &clientcmdapi.Context{ + Cluster: "docker-desktop", // reuse existing cluster + AuthInfo: "docker-desktop", // reuse existing auth + } - err = clientcmd.WriteToFile(*conf, "./test_data/kubeconfig3") - require.NoError(t, err) + // Write back to file + err = clientcmd.WriteToFile(*config, "./test_data/kubeconfig1") + require.NoError(t, err) - t.Log("created kubeconfig3 file") + // Wait for context to be added + found := false - // check if kubeconfig3 is loaded - context, err := kubeConfigStore.GetContext("random-cluster-4") + for i := 0; i < 20; i++ { + context, err := kubeConfigStore.GetContext("random-cluster-4") + if err == nil && context != nil { + found = true + break + } - // loop for until GetContext returns "random-cluster-4" or 30 seconds has past - for i := 0; i < 30; i++ { - if err == nil && context.Name == "random-cluster-4" { - break + time.Sleep(500 * time.Millisecond) } - time.Sleep(1 * time.Second) + require.True(t, found, "Context should have been added") + }) - context, err = kubeConfigStore.GetContext("random-cluster-4") - } + // Test removing a context + t.Run("Remove context", func(t *testing.T) { + // Verify context exists before removal + context, err := kubeConfigStore.GetContext("random-cluster-4") + require.NoError(t, err) + require.NotNil(t, context) + + // Read existing config + config, err := clientcmd.LoadFromFile("./test_data/kubeconfig1") + require.NoError(t, err) + + // Remove context + delete(config.Contexts, "random-cluster-4") + + // Write back to file + err = clientcmd.WriteToFile(*config, "./test_data/kubeconfig1") + require.NoError(t, err) - require.NoError(t, err) - require.Equal(t, "random-cluster-4", context.Name) + // Wait for context to be removed + removed := false - // delete kubeconfig3 file - err = os.Remove("./test_data/kubeconfig3") - require.NoError(t, err) + for i := 0; i < 20; i++ { + _, err = kubeConfigStore.GetContext("random-cluster-4") + if err != nil { + removed = true + break + } + + time.Sleep(500 * time.Millisecond) + } + + require.True(t, removed, "Context should have been removed") + }) + + // Cleanup in case test fails + defer func() { + config, err := clientcmd.LoadFromFile("./test_data/kubeconfig1") + if err == nil { + delete(config.Contexts, "random-cluster-4") + + err = clientcmd.WriteToFile(*config, "./test_data/kubeconfig1") + require.NoError(t, err) + } + }() }