Implement mass machine type transition job WIP

Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:alpine
+RUN mkdir /app
+ADD . /app/
+WORKDIR /app
+RUN go build -o main .
+RUN adduser -S -D -H -h /app appuser
+USER appuser
+CMD ["./main"]

README.md

@@ -1,2 +1,5 @@
 # Mass machine type transition
 This project contains the k8s job that will perform a mass machine type transition on an OpenShift cluster.
+
+## Notes
+This is a more cohesive version of the mass machine type transition. This implementation uses Docker to create an image from the golang script that performs the machine type transition, and hangs until all warnings are cleared. The .yaml file is a job that uses this generated image, and will only be completed once this script exits, i.e. there are no warning labels left on any VMs. To run this locally, you will need to build a Docker image with the tag "machine-type-transition", then start the job with kubectl apply. This is not comprehensively tested yet; in the future I will be automating the process of running this job, as well as adding functionality for the user to specify flags to update machine types only on specific namespaces, and to force a restart/shut down of all running VMs when updating their machine types.

informers.go

@@ -1,14 +1,25 @@
 package mass_machine_type_transition
 
 import (
+	"fmt"
+	"strings"
+	"time"
+
 	k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
 	k6tv1 "kubevirt.io/api/core/v1"
 	"kubevirt.io/client-go/kubecli"
-	"time"
 )
 
+// using this as a const allows us to easily modify the program to update if a newer version is released
+// we generally want to be updating the machine types to the most recent version
+const latestVersion = "rhel9.2.0"
+
+var vmiList = []string{}
+var exitJob = make(chan struct{})
+
 func getVirtCli() (kubecli.KubevirtClient, error) {
 	clientConfig, err := kubecli.GetKubevirtClientConfig()
 	if err != nil {
@@ -23,14 +34,89 @@ func getVirtCli() (kubecli.KubevirtClient, error) {
 	return virtCli, err
 }
 
-func getVmiInformer() (cache.SharedIndexInformer, error) {
-	virtCli, err := getVirtCli()
-	if err != nil {
-		return nil, err
-	}
-
+func getVmiInformer(virtCli kubecli.KubevirtClient) (cache.SharedIndexInformer, error) {
 	listWatcher := cache.NewListWatchFromClient(virtCli.RestClient(), "virtualmachineinstances", k8sv1.NamespaceAll, fields.Everything())
 	vmiInformer := cache.NewSharedIndexInformer(listWatcher, &k6tv1.VirtualMachineInstance{}, 1*time.Hour, cache.Indexers{})
+
+	vmiInformer.AddEventHandler(cache.ResourceEventHandlerFuncs {
+		DeleteFunc: removeWarningLabel,
+	})
 
 	return vmiInformer, nil
 }
+
+func updateMachineTypes(virtCli kubecli.KubevirtClient) error {
+	vmList, err := virtCli.VirtualMachine(k8sv1.NamespaceAll).List(&k8sv1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, vm := range vmList.Items {
+		machineType := vm.Spec.Template.Spec.Domain.Machine.Type
+		machineTypeSubstrings := strings.Split(machineType, "-")
+		version := machineTypeSubstrings[2]
+		if len(machineTypeSubstrings) != 3 {
+			return nil
+		}
+
+		if strings.Contains(version, "rhel") && version < "rhel9.0.0" {
+			machineTypeSubstrings[2] = latestVersion
+			machineType = strings.Join(machineTypeSubstrings, "-")
+			updateMachineType := fmt.Sprintf(`{"spec": {"template": {"spec": {"domain": {"machine": {"type": "%s"}}}}}}`, machineType)
+
+			_, err = virtCli.VirtualMachine(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(updateMachineType), &k8sv1.PatchOptions{})
+			if err != nil {
+				return err
+			}
+
+			// add label to running VMs that a restart is required for change to take place
+			if vm.Status.Ready {
+				err = addWarningLabel(virtCli, &vm)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func addWarningLabel (virtCli kubecli.KubevirtClient, vm *k6tv1.VirtualMachine) error {
+	addLabel := fmt.Sprint(`{"metadata": {"labels": {"restart-vm-required": "true"}}}}}`)
+	_, err := virtCli.VirtualMachineInstance(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(addLabel), &k8sv1.PatchOptions{})
+	if err != nil {
+		return err
+	}
+	vmiList = append(vmiList, vm.Name)
+
+	return nil
+}
+
+func removeWarningLabel(obj interface{}) {
+	vmi, ok := obj.(*k6tv1.VirtualMachineInstance)
+	if !ok {
+		return
+	}
+
+	//check if deleted VMI is in list of VMIs that need to be restarted
+	vmiIndex := searchVMIList(vmi.Name)
+	if  vmiIndex == -1 {
+		return
+	}
+
+	// remove deleted VMI from list
+	vmiList = append(vmiList[:vmiIndex], vmiList[vmiIndex+1:]...)
+
+	// check if VMI list is now empty, to signal exiting the job
+	if len(vmiList) == 0 {
+		close(exitJob)
+	}
+}
+
+func searchVMIList(vmiName string) int {
+	for i, element := range vmiList {
+		if element == vmiName {
+			return i
+		}
+	}
+	return -1
+}

main.go

@@ -3,14 +3,24 @@ package mass_machine_type_transition
 import "os"
 
 func main() {
-	vmiInformer, err := getVmiInformer()
+	virtCli, err := getVirtCli()
 	if err != nil {
 		os.Exit(1)
 	}
-
-	if vmiInformer != nil {
+
+	vmiInformer, err := getVmiInformer(virtCli)
+	if err != nil {
 		os.Exit(1)
 	}
-
+
+	stopCh := make(chan struct{})
+	go vmiInformer.Run(stopCh)
+
+
+	updateMachineTypes(virtCli)
+
+	// wait for list of VMIs that need restart to be empty
+	<-exitJob
+
 	os.Exit(0)
 }

mass_machine_type_transition_job.yaml

@@ -0,0 +1,14 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mass-machine-type-transition
+spec:
+  template:
+    metadata:
+      name: mass-machine-type-transition
+    spec:
+      containers:
+      - name: machine-type-transition
+        image: machine-type-transition:latest
+        imagePullPolicy: IfNotPresent
+      restartPolicy: OnFailure

massmachinetypejob.go

@@ -0,0 +1,45 @@
+package mass_machine_type_transition
+/*
+import (
+	"context"
+	//"strings"
+
+	batchv1 "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
+	k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"kubevirt.io/client-go/kubecli"
+)
+
+
+// leftover from nested job approach
+func generateMachineTypeTransitionJob(virtCli kubecli.KubevirtClient, namespace *string, completionCount *int32) error {
+	jobs := virtCli.BatchV1().Jobs("default")
+	jobSpec := &batchv1.Job{
+		ObjectMeta: k8sv1.ObjectMeta{
+			Name:      "mass-machine-type-transition",
+			Namespace: *namespace,
+		},
+		Spec: batchv1.JobSpec{
+			Completions: completionCount,
+			Template:    v1.PodTemplateSpec{
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+					    {
+						Name:    "mass-machine-type-transition",
+						// not 100% sure what to use for image here
+						Image:   "",
+						Command: "",
+					    },
+					},
+					RestartPolicy: v1.RestartPolicyOnFailure,
+				},
+			},
+		},
+	}
+
+	_, err := jobs.Create(context.TODO(), jobSpec, k8sv1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}*/