Implement mass machine type transition job WIP

Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:alpine
+RUN mkdir /app
+ADD . /app/
+WORKDIR /app
+RUN go build -o main .
+RUN adduser -S -D -H -h /app appuser
+USER appuser
+CMD ["./main"]

README.md

@@ -1,2 +1,5 @@
 # Mass machine type transition
 This project contains the k8s job that will perform a mass machine type transition on an OpenShift cluster.
+
+## Notes
+This is a more cohesive version of the mass machine type transition. This implementation uses Docker to create an image from the golang script that performs the machine type transition, and hangs until all warnings are cleared. The .yaml file is a job that uses this generated image, and will only be completed once this script exits, i.e. there are no warning labels left on any VMs. To run this locally, you will need to build a Docker image with the tag "machine-type-transition", then start the job with kubectl apply. This is not comprehensively tested yet; in the future I will be automating the process of running this job, as well as adding functionality for the user to specify flags to update machine types only on specific namespaces, and to force a restart/shut down of all running VMs when updating their machine types.

informers.go

@@ -1,12 +1,15 @@
 package mass_machine_type_transition
 
 import (
+	"fmt"
+	"time"
+
 	k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
 	k6tv1 "kubevirt.io/api/core/v1"
 	"kubevirt.io/client-go/kubecli"
-	"time"
 )
 
 func getVirtCli() (kubecli.KubevirtClient, error) {
@@ -23,14 +26,64 @@ func getVirtCli() (kubecli.KubevirtClient, error) {
 	return virtCli, err
 }
 
-func getVmiInformer() (cache.SharedIndexInformer, error) {
-	virtCli, err := getVirtCli()
-	if err != nil {
-		return nil, err
-	}
-
-	listWatcher := cache.NewListWatchFromClient(virtCli.RestClient(), "virtualmachineinstances", k8sv1.NamespaceAll, fields.Everything())
+func getVmiInformer(virtCli kubecli.KubevirtClient) (cache.SharedIndexInformer, error) {
+	listWatcher := cache.NewListWatchFromClient(virtCli.RestClient(), "virtualmachineinstances", namespace, fields.Everything())
 	vmiInformer := cache.NewSharedIndexInformer(listWatcher, &k6tv1.VirtualMachineInstance{}, 1*time.Hour, cache.Indexers{})
+
+	vmiInformer.AddEventHandler(cache.ResourceEventHandlerFuncs {
+		DeleteFunc: handleDeletedVmi,
+	})
 
 	return vmiInformer, nil
 }
+
+func handleDeletedVmi(obj interface{}) {
+	vmi, ok := obj.(*k6tv1.VirtualMachineInstance)
+	if !ok {
+		return
+	}
+
+	// get VMI name in the format namespace/name
+	vmiKey, err := cache.MetaNamespaceKeyFunc(vmi)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// check if deleted VMI is in list of VMIs that need to be restarted
+	_, exists := vmisPendingUpdate[vmiKey]
+	if  !exists {
+		return
+	}
+
+	// remove warning label from VM
+	// if removing the warning label fails, exit before removing VMI from list
+	// since the label is still there to tell the user to restart, it wouldn't
+	// make sense to have a mismatch between the number of VMs with the label
+	// and the number of VMIs in the list of VMIs pending update.
+
+	err = removeWarningLabel(vmi)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// remove deleted VMI from list
+	delete(vmisPendingUpdate, vmiKey)
+
+	// check if VMI list is now empty, to signal exiting the job
+	if len(vmisPendingUpdate) == 0 {
+		close(exitJob)
+	}
+}
+
+func removeWarningLabel(vmi *k6tv1.VirtualMachineInstance) error {
+	virtCli, err := getVirtCli()
+	if err != nil {
+		return err
+	}
+
+	removeLabel := fmt.Sprint(`{"op": "remove", "path": "/metadata/labels/restart-vm-required"}`)
+	_, err = virtCli.VirtualMachine(vmi.Namespace).Patch(vmi.Name, types.JSONPatchType, []byte(removeLabel), &k8sv1.PatchOptions{})
+	return err
+}

main.go

@@ -1,16 +1,48 @@
 package mass_machine_type_transition
 
-import "os"
+import (
+	"fmt"
+	"os"
+	"strconv"
+)
 
 func main() {
-	vmiInformer, err := getVmiInformer()
+	var err error
+	// update restartNow if env is set
+	restartEnv, exists := os.LookupEnv("FORCE_RESTART")
+	if exists {
+		restartNow, err = strconv.ParseBool(restartEnv)
+		if err != nil {
+			fmt.Println(err)
+		}
+	}
+
+	// update namespace if env is set
+	namespaceEnv, exists := os.LookupEnv("NAMESPACE")
+	if exists {
+		namespace = namespaceEnv
+	}
+
+	virtCli, err := getVirtCli()
 	if err != nil {
 		os.Exit(1)
 	}
-
-	if vmiInformer != nil {
+
+	vmiInformer, err := getVmiInformer(virtCli)
+	if err != nil {
 		os.Exit(1)
 	}
-
+
+	go vmiInformer.Run(exitJob)
+
+
+	err = updateMachineTypes(virtCli)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	// wait for list of VMIs that need restart to be empty
+	<-exitJob
+
 	os.Exit(0)
 }

mass_machine_type_transition_job.yaml

@@ -0,0 +1,19 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mass-machine-type-transition
+spec:
+  template:
+    metadata:
+      name: mass-machine-type-transition
+    spec:
+      containers:
+      - name: machine-type-transition
+        image: machine-type-transition:latest
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: NAMESPACE
+          value: ""
+        - name: FORCE_RESTART
+          value: false
+      restartPolicy: OnFailure

update_machine_type.go

@@ -0,0 +1,109 @@
+package mass_machine_type_transition
+
+import (
+	"fmt"
+	"strings"
+
+	k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/cache"
+	k6tv1 "kubevirt.io/api/core/v1"
+	"kubevirt.io/client-go/kubecli"
+)
+
+// using this as a const allows us to easily modify the program to update if a newer version is released
+// we generally want to be updating the machine types to the most recent version
+const latestMachineTypeVersion = "rhel9.2.0"
+
+var (
+	vmisPendingUpdate = make(map[string]struct{})
+	exitJob = make(chan struct{})
+
+	// by default, update machine type across all namespaces
+	namespace = k8sv1.NamespaceAll
+	//labels []string
+
+	// by default, should require manual restarting of VMIs
+	restartNow = false
+)
+
+func patchVmMachineType(virtCli kubecli.KubevirtClient, vm *k6tv1.VirtualMachine, machineType string) error {
+	updateMachineType := fmt.Sprintf(`{"spec": {"template": {"spec": {"domain": {"machine": {"type": "%s"}}}}}}`, machineType)
+
+	_, err := virtCli.VirtualMachine(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(updateMachineType), &k8sv1.PatchOptions{})
+	if err != nil {
+		return err
+	}
+
+	// add label to running VMs that a restart is required for change to take place
+	if vm.Status.Created {
+		// adding the warning label to the VMs regardless if we restart them now or if the user does it manually
+		// shouldn't matter, since the deletion of the VMI will remove the label and remove the vmi list anyway
+		err = addWarningLabel(virtCli, vm)
+		if err != nil {
+			return err
+		}
+
+		if restartNow {
+			err = virtCli.VirtualMachine(vm.Namespace).Restart(vm.Name, &k6tv1.RestartOptions{})
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func updateMachineTypes(virtCli kubecli.KubevirtClient) error {
+	vmList, err := virtCli.VirtualMachine(namespace).List(&k8sv1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, vm := range vmList.Items {
+		machineType := vm.Spec.Template.Spec.Domain.Machine.Type
+		machineTypeSubstrings := strings.Split(machineType, "-")
+
+		// in the case where q35 is the machine type, the VMI status will contain the
+		// full machine type string. Since q35 is an alias for the most recent machine type,
+		// if a VM hasn't been restarted and thus still has an outdated machine type, it should
+		// be updated to the most recent machine type. I'm not able to access the machine type from
+		// VMI status using the kv library, so instead I am opting to update the machine types of all
+		// VMs with q35 instead. It might only be necessary to update the ones that are running as well,
+		// though I am not sure.
+
+		if len(machineTypeSubstrings) == 1 {
+			if machineTypeSubstrings[0] == "q35" {
+				machineType = "pc-q35-" + latestMachineTypeVersion
+				return patchVmMachineType(virtCli, &vm, machineType)
+			}
+		}
+
+		if len(machineTypeSubstrings) == 3 {
+			version := machineTypeSubstrings[2]
+			if strings.Contains(version, "rhel") && version < "rhel9.0.0" {
+				machineTypeSubstrings[2] = latestMachineTypeVersion
+				machineType = strings.Join(machineTypeSubstrings, "-")
+				return patchVmMachineType(virtCli, &vm, machineType)
+			}
+		}
+	}
+	return nil
+}
+
+func addWarningLabel (virtCli kubecli.KubevirtClient, vm *k6tv1.VirtualMachine) error {
+	addLabel := fmt.Sprint(`{"metadata": {"labels": {"restart-vm-required": "true"}}}}}`)
+	_, err := virtCli.VirtualMachine(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(addLabel), &k8sv1.PatchOptions{})
+	if err != nil {
+		return err
+	}
+
+	// get VM name in the format namespace/name
+	vmKey, err := cache.MetaNamespaceKeyFunc(vm)
+	if err != nil {
+		return err
+	}
+	vmisPendingUpdate[vmKey] = struct{}{}
+
+	return nil
+}