From 61146f87fbdfd3655f4e78249c4d1ab96e822d69 Mon Sep 17 00:00:00 2001 From: Tchoupinax Date: Tue, 7 May 2024 22:43:24 +0200 Subject: [PATCH] feat: handle Keda scaled object --- main.go | 26 +++++++++------ pod-status.go | 5 +-- resources/keda.go | 69 ++++++++++++++++++++++++++++++++++++++++ steps.go | 59 +++++++++++++++++++--------------- summary.go | 20 ++++++------ utils.go | 23 +++++++++++++- logs.go => utils/logs.go | 14 ++++---- 7 files changed, 160 insertions(+), 56 deletions(-) create mode 100644 resources/keda.go rename logs.go => utils/logs.go (76%) diff --git a/main.go b/main.go index 4c1e575..2230647 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,9 @@ import ( "flag" "os" + utils "github.com/Tchoupinax/k8s-labels-migrator/utils" istio "istio.io/client-go/pkg/clientset/versioned" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" ) @@ -20,7 +22,7 @@ func main() { clientset, err := kubernetes.NewForConfig(config) istioClient, err := istio.NewForConfig(config) - //crdClient, err := dynamic.NewForConfig(config) + crdClient, err := dynamic.NewForConfig(config) if err != nil { panic(err.Error()) } @@ -30,6 +32,11 @@ func main() { var labelToChangeKey = "" var labelToChangeValue = "" var goalOfOperationIsToRemoveLabel = false + var matcherLabels = []string{ + "app.kubernetes.io/instance", + "app.kubernetes.io/name", + "app.kubernetes.io/version", + } flag.StringVar(&deploymentName, "deployment", "", "Name of the deployment to edit label") flag.StringVar(&namespace, "namespace", "", "Namespace of the deployment to edit label") @@ -39,21 +46,21 @@ func main() { flag.Parse() if deploymentName == "" { - logError("Deployment name is mandatory") + utils.LogError("Deployment name is mandatory") os.Exit(1) } if namespace == "" { - logError("Namespace is mandatory") + utils.LogError("Namespace is mandatory") os.Exit(1) } if labelToChangeValue == "" && !goalOfOperationIsToRemoveLabel { - logError("label value is mandatory") + utils.LogError("label value is mandatory") os.Exit(1) } - logInfo("Analyzing your cluster...") + utils.LogInfo("Analyzing your cluster...") resourcesAnalyze(clientset, istioClient, namespace, deploymentName, labelToChangeKey) - logSuccess("Cluster ready") + utils.LogSuccess("Cluster ready") displaySummary( namespace, deploymentName, @@ -64,12 +71,12 @@ func main() { c := askForConfirmation("Do you validate these parameters?") if !c { - logInfo("Operation aborted by the user") + utils.LogInfo("Operation aborted by the user") os.Exit(0) } c2 := askForConfirmation("I confirm that I have no gitops tool overriding my config (e.g. ArgoCD auto-sync)") if !c2 { - logInfo("Operation aborted by the user") + utils.LogInfo("Operation aborted by the user") os.Exit(0) } @@ -77,13 +84,14 @@ func main() { namespace, clientset, istioClient, + crdClient, deploymentName, labelToChangeKey, labelToChangeValue, goalOfOperationIsToRemoveLabel, ) - if labelToChangeKey == "app.kubernetes.io/name" { + if arrayContains(matcherLabels, labelToChangeKey) { AddLabelToServiceSelector( namespace, clientset, diff --git a/pod-status.go b/pod-status.go index 44949c8..079a22d 100644 --- a/pod-status.go +++ b/pod-status.go @@ -3,6 +3,7 @@ package main import ( "context" + utils "github.com/Tchoupinax/k8s-labels-migrator/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -14,9 +15,9 @@ func waitUntilAllPodAreReady( ) bool { deployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) if err != nil { - logError(err.Error()) + utils.LogError(err.Error()) return false } - currentDeploymentHasAllPodReady := deployment.Status.Replicas-deployment.Status.ReadyReplicas == 0 && deployment.Status.Replicas > 1 + currentDeploymentHasAllPodReady := deployment.Status.Replicas-deployment.Status.ReadyReplicas == 0 && deployment.Status.Replicas > 0 return currentDeploymentHasAllPodReady } diff --git a/resources/keda.go b/resources/keda.go new file mode 100644 index 0000000..6e28750 --- /dev/null +++ b/resources/keda.go @@ -0,0 +1,69 @@ +package keda + +import ( + "context" + "fmt" + "time" + + utils "github.com/Tchoupinax/k8s-labels-migrator/utils" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// https://keda.sh/docs/2.13/concepts/scaling-deployments + +func PauseScaledObject( + crdClient *dynamic.DynamicClient, + clientset *kubernetes.Clientset, + deploymentName string, + namespace string, +) { + crdGVR := schema.GroupVersionResource{ + Group: "keda.sh", + Version: "v1alpha1", + Resource: "scaledobjects", + } + kedaScaledObject, _ := crdClient.Resource(crdGVR).Namespace(namespace).Get(context.TODO(), deploymentName, v1.GetOptions{}) + if kedaScaledObject != nil { + utils.LogInfo("Keda Scaled Object detected") + // Add the annotation "autoscaling.keda.sh/paused" + kedaScaledObject.Object["metadata"].(map[string]interface{})["annotations"].(map[string]interface{})["autoscaling.keda.sh/paused"] = "true" + crdClient.Resource(crdGVR).Namespace(namespace).Update(context.TODO(), kedaScaledObject, v1.UpdateOptions{}) + utils.LogSuccess("Keda object paused ⏸️") + utils.LogBlocking("Waiting randomly 5 seconds to ensure keda controller registered the update") + for range 5 { + utils.LogBlockingDot() + time.Sleep(1 * time.Second) + } + fmt.Println() + } else { + utils.LogInfo("Any Keda Scaled Object detected") + } +} + +func ResumeScaledObject( + crdClient *dynamic.DynamicClient, + clientset *kubernetes.Clientset, + deploymentName string, + namespace string, +) { + crdGVR := schema.GroupVersionResource{ + Group: "keda.sh", + Version: "v1alpha1", + Resource: "scaledobjects", + } + kedaScaledObject, _ := crdClient.Resource(crdGVR).Namespace(namespace).Get(context.TODO(), deploymentName, v1.GetOptions{}) + if kedaScaledObject != nil { + utils.LogInfo("Keda Scaled Object detected") + delete( + kedaScaledObject.Object["metadata"].(map[string]interface{})["annotations"].(map[string]interface{}), + "autoscaling.keda.sh/paused", + ) + utils.LogSuccess("Keda object resumed ▶️") + crdClient.Resource(crdGVR).Namespace(namespace).Update(context.TODO(), kedaScaledObject, v1.UpdateOptions{}) + } else { + utils.LogInfo("Any Keda Scaled Object detected") + } +} diff --git a/steps.go b/steps.go index 93d42ea..3f4d8cb 100644 --- a/steps.go +++ b/steps.go @@ -7,9 +7,12 @@ import ( "strings" "time" + keda "github.com/Tchoupinax/k8s-labels-migrator/resources" + utils "github.com/Tchoupinax/k8s-labels-migrator/utils" istio "istio.io/client-go/pkg/clientset/versioned" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" ) @@ -17,6 +20,7 @@ func MigrationWorkflow( namespace string, clientset *kubernetes.Clientset, istioClient *istio.Clientset, + crdClient *dynamic.DynamicClient, deploymentName string, changingLabelKey string, changingLabelValue string, @@ -26,11 +30,11 @@ func MigrationWorkflow( currentDeployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) currentDestinationRule, _ := istioClient.NetworkingV1alpha3().DestinationRules(namespace).Get(context.TODO(), deploymentName, v1.GetOptions{}) if err != nil { - logError("No deployment found.") + utils.LogError("No deployment found.") os.Exit(1) } - logInfo("1. Creating the clone deployment") + utils.LogInfo("1. Creating the clone deployment") var temporalDeployment = *currentDeployment temporalDeployment.GenerateName = fmt.Sprintf("%s-%s", currentDeployment.Name, "changing-label-tmp") temporalDeployment.Name = fmt.Sprintf("%s-%s", currentDeployment.Name, "changing-label-tmp") @@ -40,30 +44,33 @@ func MigrationWorkflow( _, err = clientset.AppsV1().Deployments(namespace).Create(context.TODO(), &temporalDeployment, metav1.CreateOptions{}) if err != nil { if strings.Contains(err.Error(), "already exists, the server was not able to generate a unique name for the object") { - logWarning("1. Temporary deployment already created. Continue...") + utils.LogWarning("1. Temporary deployment already created. Continue...") } } else { - logSuccess("1. Deployment replicated") + utils.LogSuccess("1. Deployment replicated") } - logInfo("2. Updating the service...") + utils.LogInfo("2. Updating the service...") var temporalService = *currentService delete(temporalService.Spec.Selector, changingLabelKey) _, err = clientset.CoreV1().Services(namespace).Update(context.TODO(), &temporalService, metav1.UpdateOptions{}) check(err) - logSuccess("2. Service updated") + utils.LogSuccess("2. Service updated") - logInfo("2.1 Updating Istio destination rules...") + utils.LogInfo("2.1 Updating Istio destination rules...") var temporalDestinationRule = *currentDestinationRule delete(temporalDestinationRule.Spec.Subsets[0].Labels, changingLabelKey) _, err = istioClient.NetworkingV1alpha3().DestinationRules(namespace).Update(context.TODO(), &temporalDestinationRule, metav1.UpdateOptions{}) check(err) - logSuccess("2.1 Istio destination rules updated") + utils.LogSuccess("2.1 Istio destination rules updated") - logBlocking("3. Waiting while pods are not totally ready to handle traffic") + utils.LogInfo("2.2 Keda") + keda.PauseScaledObject(crdClient, clientset, deploymentName, namespace) + + utils.LogBlocking("3. Waiting while pods are not totally ready to handle traffic") areAllPodReady := false for !areAllPodReady { - logBlockingDot() + utils.LogBlockingDot() time.Sleep(1 * time.Second) areAllPodReady = waitUntilAllPodAreReady(clientset, namespace, currentDeployment.Name) && @@ -71,13 +78,13 @@ func MigrationWorkflow( } fmt.Println("") - logInfo("4. Delete the old deployment...") + utils.LogInfo("4. Delete the old deployment...") // Delete the old deployment deleteError := clientset.AppsV1().Deployments(namespace).Delete(context.TODO(), currentDeployment.Name, *metav1.NewDeleteOptions(0)) check(deleteError) - logSuccess("4. Old deployment deleted") + utils.LogSuccess("4. Old deployment deleted") - logInfo("5. Creating the original deployment with modified label") + utils.LogInfo("5. Creating the original deployment with modified label") var futureOfficialDeployment = *currentDeployment futureOfficialDeployment.GenerateName = deploymentName futureOfficialDeployment.Name = deploymentName @@ -106,23 +113,23 @@ func MigrationWorkflow( } fmt.Println(err) } - logSuccess("5. Deployment created") + utils.LogSuccess("5. Deployment created") - logBlocking("6. Waiting while pods are not totally ready to handle traffic") + utils.LogBlocking("6. Waiting while pods are not totally ready to handle traffic") areAllPodReady = false for !areAllPodReady { - logBlockingDot() + utils.LogBlockingDot() time.Sleep(1 * time.Second) areAllPodReady = waitUntilAllPodAreReady(clientset, namespace, currentDeployment.Name) } fmt.Println("") - logInfo("7. Deleting temporal deployment...") + utils.LogInfo("7. Deleting temporal deployment...") time.Sleep(1 * time.Second) // Delete the temporal deployment errDeleteTmpDeploy := clientset.AppsV1().Deployments(namespace).Delete(context.TODO(), fmt.Sprintf("%s-%s", currentDeployment.Name, "changing-label-tmp"), metav1.DeleteOptions{}) check(errDeleteTmpDeploy) - logSuccess("7. Temporary deployment deleted") + utils.LogSuccess("7. Temporary deployment deleted") } func AddLabelToServiceSelector( @@ -133,8 +140,8 @@ func AddLabelToServiceSelector( changingLabelValue string, removeLabel bool, ) { - logInfo("====== Additionnal step ====================================") - logInfo("8. Add the label as a selector in the service...") + utils.LogInfo("====== Additionnal step ====================================") + utils.LogInfo("8. Add the label as a selector in the service...") // Get the current service currentService, err := clientset.CoreV1().Services(namespace).Get(context.TODO(), applicationName, metav1.GetOptions{}) check(err) @@ -152,8 +159,8 @@ func AddLabelToServiceSelector( _, updateServiceError := clientset.CoreV1().Services(namespace).Update(context.TODO(), &futureService, metav1.UpdateOptions{}) check(updateServiceError) - logSuccess("8. Service configured") - logInfo("============================================================") + utils.LogSuccess("8. Service configured") + utils.LogInfo("============================================================") } func AddLabelToIstioDestinatonRulesSelector( @@ -165,8 +172,8 @@ func AddLabelToIstioDestinatonRulesSelector( changingLabelValue string, removeLabel bool, ) { - logInfo("====== Additionnal step ====================================") - logInfo("9. Add the label as a selector in istio destination rules...") + utils.LogInfo("====== Additionnal step ====================================") + utils.LogInfo("9. Add the label as a selector in istio destination rules...") currentDestinationRule, err := istioClient.NetworkingV1alpha3().DestinationRules(namespace).Get(context.TODO(), applicationName, v1.GetOptions{}) check(err) @@ -183,6 +190,6 @@ func AddLabelToIstioDestinatonRulesSelector( _, updateDestinationRuleError := istioClient.NetworkingV1alpha3().DestinationRules(namespace).Update(context.TODO(), &futureDestinationRule, metav1.UpdateOptions{}) check(updateDestinationRuleError) - logSuccess("9. Istio destination rules configured") - logInfo("============================================================") + utils.LogSuccess("9. Istio destination rules configured") + utils.LogInfo("============================================================") } diff --git a/summary.go b/summary.go index b3fc919..40a1bc8 100644 --- a/summary.go +++ b/summary.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "os" + "strings" + utils "github.com/Tchoupinax/k8s-labels-migrator/utils" "github.com/jedib0t/go-pretty/v6/table" istio "istio.io/client-go/pkg/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,7 +53,7 @@ func resourcesAnalyze( // Version: "v1alpha1", // Resource: "scaledobjects", //} - //kedaScaledObject, _ := crdClient.Resource(crdGVR).Namespace(namespace).Get(context.TODO(), "account-contract-live", v1.GetOptions{}) + //kedaScaledObject, _ := crdClient.Resource(crdGVR).Namespace(namespace).Get(context.TODO(), "", v1.GetOptions{}) deploymentSelectorLabels := deployment.Spec.Template.ObjectMeta.Labels serviceSelectorLabels := service.Spec.Selector @@ -62,12 +64,13 @@ func resourcesAnalyze( fmt.Println() t := table.NewWriter() t.SetOutputMirror(os.Stdout) - t.AppendHeader(table.Row{"Type", "Name", "Detected", "labels count", "valid"}) + t.AppendHeader(table.Row{"Type", "Name", "Detected", "labels count", "labels", "valid"}) t.AppendRows([]table.Row{{ "Deployment", If(deployment.Name != "", deployment.Name, "—"), If(deployment != nil, "✅", "❌"), len(deploymentSelectorLabels), + strings.Join(mapToArray(deploymentSelectorLabels), "\n"), If(len(deploymentSelectorLabels) == 1 && deploymentSelectorLabels[changingLabelKey] != "", "❌", "✅"), }}) t.AppendRows([]table.Row{{ @@ -75,6 +78,7 @@ func resourcesAnalyze( If(service.Name != "", service.Name, "—"), If(service.Name != "", "✅", "❌"), len(serviceSelectorLabels), + strings.Join(mapToArray(serviceSelectorLabels), "\n"), If(len(serviceSelectorLabels) == 1 && serviceSelectorLabels[changingLabelKey] != "", "❌", "✅"), }}) t.AppendRows([]table.Row{{ @@ -82,13 +86,7 @@ func resourcesAnalyze( If(service.Name != "", destinationRule.Name, "—"), If(service.Name != "", "✅", "❌"), len(destinationRuleSelectorLabels), - If(len(destinationRuleSelectorLabels) == 1 && destinationRuleSelectorLabels[changingLabelKey] != "", "❌", "✅"), - }}) - t.AppendRows([]table.Row{{ - " DestinationRule", - If(service.Name != "", destinationRule.Name, "—"), - If(service.Name != "", "✅", "❌"), - len(destinationRuleSelectorLabels), + strings.Join(mapToArray(destinationRuleSelectorLabels), "\n"), If(len(destinationRuleSelectorLabels) == 1 && destinationRuleSelectorLabels[changingLabelKey] != "", "❌", "✅"), }}) t.SetStyle(table.StyleColoredBright) @@ -96,12 +94,12 @@ func resourcesAnalyze( fmt.Println() if len(deploymentSelectorLabels) == 1 && deploymentSelectorLabels[changingLabelKey] != "" { - logError(fmt.Sprintf("The label \"%s\" can not be edited because it's the only one in the matching set for the deployment", changingLabelKey)) + utils.LogError(fmt.Sprintf("The label \"%s\" can not be edited because it's the only one in the matching set for the deployment", changingLabelKey)) os.Exit(1) } if len(serviceSelectorLabels) == 1 && serviceSelectorLabels[changingLabelKey] != "" { - logError(fmt.Sprintf("The label \"%s\" can not be edited because it's the only one in the matching set for the service", changingLabelKey)) + utils.LogError(fmt.Sprintf("The label \"%s\" can not be edited because it's the only one in the matching set for the service", changingLabelKey)) os.Exit(1) } } diff --git a/utils.go b/utils.go index 0aab513..4ce36b8 100644 --- a/utils.go +++ b/utils.go @@ -5,7 +5,10 @@ import ( "fmt" "log" "os" + "slices" "strings" + + utils "github.com/Tchoupinax/k8s-labels-migrator/utils" ) func check(e error) { @@ -28,7 +31,7 @@ func askForConfirmation(s string) bool { reader := bufio.NewReader(os.Stdin) for { - logWarning(fmt.Sprintf("%s [y/n]: ", s)) + utils.LogWarning(fmt.Sprintf("%s [y/n]: ", s)) response, err := reader.ReadString('\n') if err != nil { @@ -51,3 +54,21 @@ func If[T any](cond bool, vtrue, vfalse T) T { } return vfalse } + +func mapToArray(myMap map[string]string) []string { + v := make([]string, 0, len(myMap)) + for key, value := range myMap { + v = append(v, key+"="+value) + } + slices.Sort(v) + return v +} + +func arrayContains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} diff --git a/logs.go b/utils/logs.go similarity index 76% rename from logs.go rename to utils/logs.go index e1d6a88..3f3d94e 100644 --- a/logs.go +++ b/utils/logs.go @@ -1,4 +1,4 @@ -package main +package utils import ( "fmt" @@ -6,32 +6,32 @@ import ( "github.com/fatih/color" ) -func logSuccess(msg string) { +func LogSuccess(msg string) { green := color.New(color.Bold, color.FgGreen).SprintFunc() fmt.Println(green(fmt.Sprintf("✅ %s", msg))) } -func logInfo(msg string) { +func LogInfo(msg string) { blue := color.New(color.Bold, color.FgCyan).SprintFunc() fmt.Println(blue(fmt.Sprintf("🌱 %s", msg))) } -func logBlocking(msg string) { +func LogBlocking(msg string) { magenta := color.New(color.Bold, color.FgHiMagenta).SprintFunc() fmt.Println(magenta(fmt.Sprintf("⌛️ %s", msg))) } -func logBlockingDot() { +func LogBlockingDot() { magenta := color.New(color.Bold, color.FgHiMagenta).SprintFunc() fmt.Print(magenta(".")) } -func logError(msg string) { +func LogError(msg string) { red := color.New(color.Bold, color.FgRed).SprintFunc() fmt.Println(red(fmt.Sprintf("❌ %s", msg))) } -func logWarning(msg string) { +func LogWarning(msg string) { yellow := color.New(color.Bold, color.FgYellow).SprintFunc() fmt.Println(yellow(fmt.Sprintf("⚠️ %s", msg))) }