Implement mass machine type transition job WIP

Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:alpine
+RUN mkdir /app
+ADD . /app/
+WORKDIR /app
+RUN go build -o main .
+RUN adduser -S -D -H -h /app appuser
+USER appuser
+CMD ["./main"]

README.md

@@ -1,2 +1,5 @@
 # Mass machine type transition
 This project contains the k8s job that will perform a mass machine type transition on an OpenShift cluster.
+
+## Notes
+This is a more cohesive version of the mass machine type transition. This implementation uses Docker to create an image from the golang script that performs the machine type transition, and hangs until all warnings are cleared. The .yaml file is a job that uses this generated image, and will only be completed once this script exits, i.e. there are no warning labels left on any VMs. To run this locally, you will need to build a Docker image with the tag "machine-type-transition", then start the job with kubectl apply. This is not comprehensively tested yet; in the future I will be automating the process of running this job, as well as adding functionality for the user to specify flags to update machine types only on specific namespaces, and to force a restart/shut down of all running VMs when updating their machine types.

informers.go

@@ -1,12 +1,32 @@
 package mass_machine_type_transition
 
 import (
+	"fmt"
+	"strings"
+	"time"
+
 	k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
 	k6tv1 "kubevirt.io/api/core/v1"
 	"kubevirt.io/client-go/kubecli"
-	"time"
+)
+
+// using this as a const allows us to easily modify the program to update if a newer version is released
+// we generally want to be updating the machine types to the most recent version
+const latestMachineTypeVersion = "rhel9.2.0"
+
+var (
+	vmisPendingUpdate = make(map[string]struct{})
+	exitJob = make(chan struct{})
+
+	// by default, update machine type across all namespaces
+	namespace = k8sv1.NamespaceAll
+	//labels []string
+
+	// by default, should require manual restarting of VMIs
+	restartNow = false
 )
 
 func getVirtCli() (kubecli.KubevirtClient, error) {
@@ -23,14 +43,122 @@ func getVirtCli() (kubecli.KubevirtClient, error) {
 	return virtCli, err
 }
 
-func getVmiInformer() (cache.SharedIndexInformer, error) {
-	virtCli, err := getVirtCli()
+func getVmiInformer(virtCli kubecli.KubevirtClient) (cache.SharedIndexInformer, error) {
+	listWatcher := cache.NewListWatchFromClient(virtCli.RestClient(), "virtualmachineinstances", namespace, fields.Everything())
+	vmiInformer := cache.NewSharedIndexInformer(listWatcher, &k6tv1.VirtualMachineInstance{}, 1*time.Hour, cache.Indexers{})
+
+	vmiInformer.AddEventHandler(cache.ResourceEventHandlerFuncs {
+		UpdateFunc: handleVmiUpdate,
+		DeleteFunc: handleVmiDeletion,
+	})
+
+	return vmiInformer, nil
+}
+
+func updateMachineTypes(virtCli kubecli.KubevirtClient) error {
+	vmList, err := virtCli.VirtualMachine(namespace).List(&k8sv1.ListOptions{})
 	if err != nil {
-		return nil, err
+		return err
+	}
+	for _, vm := range vmList.Items {
+		machineType := vm.Spec.Template.Spec.Domain.Machine.Type
+		machineTypeSubstrings := strings.Split(machineType, "-")
+		version := machineTypeSubstrings[2]
+		if len(machineTypeSubstrings) != 3 {
+			return nil
+		}
+
+		if strings.Contains(version, "rhel") && version < "rhel9.0.0" {
+			machineTypeSubstrings[2] = latestMachineTypeVersion
+			machineType = strings.Join(machineTypeSubstrings, "-")
+			updateMachineType := fmt.Sprintf(`{"spec": {"template": {"spec": {"domain": {"machine": {"type": "%s"}}}}}}`, machineType)
+
+			_, err = virtCli.VirtualMachine(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(updateMachineType), &k8sv1.PatchOptions{})
+			if err != nil {
+				return err
+			}
+
+			// add label to running VMs that a restart is required for change to take place
+			if vm.Status.Ready {
+				// adding the warning label to the VMs regardless if we restart them now or if the user does it manually
+				// shouldn't matter, since the deletion of the VMI will remove the label and remove the vmi list anyway
+				err = addWarningLabel(virtCli, &vm)
+				if err != nil {
+					return err
+				}
+
+				if restartNow {
+					err = virtCli.VirtualMachine(vm.Namespace).Restart(vm.Name, &k6tv1.RestartOptions{})
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
 	}
+	return nil
+}
 
-	listWatcher := cache.NewListWatchFromClient(virtCli.RestClient(), "virtualmachineinstances", k8sv1.NamespaceAll, fields.Everything())
-	vmiInformer := cache.NewSharedIndexInformer(listWatcher, &k6tv1.VirtualMachineInstance{}, 1*time.Hour, cache.Indexers{})
+func addWarningLabel (virtCli kubecli.KubevirtClient, vm *k6tv1.VirtualMachine) error {
+	addLabel := fmt.Sprint(`{"metadata": {"labels": {"restart-vm-required": "true"}}}}}`)
+	_, err := virtCli.VirtualMachineInstance(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(addLabel), &k8sv1.PatchOptions{})
+	if err != nil {
+		return err
+	}
+
+	// get VM name in the format namespace/name
+	vmKey, err := cache.MetaNamespaceKeyFunc(vm)
+	if err != nil {
+		return err
+	}
+	vmisPendingUpdate[vmKey] = struct{}{}
+
+	return nil
+}
 
-	return vmiInformer, nil
+func handleVmiUpdate(oldObj interface{}, newObj interface{}) {
+	vmi, ok := newObj.(*k6tv1.VirtualMachineInstance)
+	if !ok {
+		return
+	}
+
+	// check VMI status for Succeeded
+	if vmi.Status.Phase != k6tv1.Succeeded {
+		return
+	}
+
+	removeWarningLabel(vmi)	
+}
+
+func handleVmiDeletion(obj interface{}) {
+	vmi, ok := obj.(*k6tv1.VirtualMachineInstance)
+	if !ok {
+		return
+	}
+
+	removeWarningLabel(vmi)
+}
+
+func removeWarningLabel(vmi *k6tv1.VirtualMachineInstance) {
+
+	// get VMI name in the format namespace/name
+	vmiKey, err := cache.MetaNamespaceKeyFunc(vmi)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	//check if deleted VMI is in list of VMIs that need to be restarted
+	_, exists := vmisPendingUpdate[vmiKey]
+	if  !exists {
+		return
+	}
+
+	// remove deleted VMI from list
+	delete(vmisPendingUpdate, vmiKey)
+
+	// check if VMI list is now empty, to signal exiting the job
+	if len(vmisPendingUpdate) == 0 {
+		close(exitJob)
+	}
 }

main.go

@@ -1,16 +1,49 @@
 package mass_machine_type_transition
 
-import "os"
+import (
+	"fmt"
+	"os"
+	"strconv"
+)
 
 func main() {
-	vmiInformer, err := getVmiInformer()
+	var err error
+	// update restartNow if env is set
+	restartEnv, exists := os.LookupEnv("RESTART_NOW")
+	if exists {
+		restartNow, err = strconv.ParseBool(restartEnv)
+		if err != nil {
+			fmt.Println(err)
+		}
+	}
+
+	// update namespace if env is set
+	namespaceEnv, exists := os.LookupEnv("NAMESPACE")
+	if exists {
+		namespace = namespaceEnv
+	}
+
+	virtCli, err := getVirtCli()
 	if err != nil {
 		os.Exit(1)
 	}
-
-	if vmiInformer != nil {
+
+	vmiInformer, err := getVmiInformer(virtCli)
+	if err != nil {
 		os.Exit(1)
 	}
-
+
+	exitJob = make(chan struct{})
+	go vmiInformer.Run(exitJob)
+
+
+	err = updateMachineTypes(virtCli)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	// wait for list of VMIs that need restart to be empty
+	<-exitJob
+
 	os.Exit(0)
 }

mass_machine_type_transition_job.yaml

@@ -0,0 +1,14 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mass-machine-type-transition
+spec:
+  template:
+    metadata:
+      name: mass-machine-type-transition
+    spec:
+      containers:
+      - name: machine-type-transition
+        image: machine-type-transition:latest
+        imagePullPolicy: IfNotPresent
+      restartPolicy: OnFailure

massmachinetypejob.go

@@ -0,0 +1,45 @@
+package mass_machine_type_transition
+/*
+import (
+	"context"
+	//"strings"
+
+	batchv1 "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
+	k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"kubevirt.io/client-go/kubecli"
+)
+
+
+// leftover from nested job approach
+func generateMachineTypeTransitionJob(virtCli kubecli.KubevirtClient, namespace *string, completionCount *int32) error {
+	jobs := virtCli.BatchV1().Jobs("default")
+	jobSpec := &batchv1.Job{
+		ObjectMeta: k8sv1.ObjectMeta{
+			Name:      "mass-machine-type-transition",
+			Namespace: *namespace,
+		},
+		Spec: batchv1.JobSpec{
+			Completions: completionCount,
+			Template:    v1.PodTemplateSpec{
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+					    {
+						Name:    "mass-machine-type-transition",
+						// not 100% sure what to use for image here
+						Image:   "",
+						Command: "",
+					    },
+					},
+					RestartPolicy: v1.RestartPolicyOnFailure,
+				},
+			},
+		},
+	}
+
+	_, err := jobs.Create(context.TODO(), jobSpec, k8sv1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}*/