diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f56836e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM golang:alpine +RUN mkdir /app +ADD . /app/ +WORKDIR /app +RUN go build -o main . +RUN adduser -S -D -H -h /app appuser +USER appuser +CMD ["./main"] diff --git a/README.md b/README.md index de07b39..aeb3aa4 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,5 @@ # Mass machine type transition This project contains the k8s job that will perform a mass machine type transition on an OpenShift cluster. + +## Notes +This is a more cohesive version of the mass machine type transition. This implementation uses Docker to create an image from the golang script that performs the machine type transition, and hangs until all warnings are cleared. The .yaml file is a job that uses this generated image, and will only be completed once this script exits, i.e. there are no warning labels left on any VMs. To run this locally, you will need to build a Docker image with the tag "machine-type-transition", then start the job with kubectl apply. This is not comprehensively tested yet; in the future I will be automating the process of running this job, as well as adding functionality for the user to specify flags to update machine types only on specific namespaces, and to force a restart/shut down of all running VMs when updating their machine types. diff --git a/informers.go b/informers.go index 36ec5ad..057308f 100644 --- a/informers.go +++ b/informers.go @@ -1,12 +1,15 @@ package mass_machine_type_transition import ( + "fmt" + "time" + k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" k6tv1 "kubevirt.io/api/core/v1" "kubevirt.io/client-go/kubecli" - "time" ) func getVirtCli() (kubecli.KubevirtClient, error) { @@ -23,14 +26,64 @@ func getVirtCli() (kubecli.KubevirtClient, error) { return virtCli, err } -func getVmiInformer() (cache.SharedIndexInformer, error) { - virtCli, err := getVirtCli() - if err != nil { - return nil, err - } - - listWatcher := cache.NewListWatchFromClient(virtCli.RestClient(), "virtualmachineinstances", k8sv1.NamespaceAll, fields.Everything()) +func getVmiInformer(virtCli kubecli.KubevirtClient) (cache.SharedIndexInformer, error) { + listWatcher := cache.NewListWatchFromClient(virtCli.RestClient(), "virtualmachineinstances", namespace, fields.Everything()) vmiInformer := cache.NewSharedIndexInformer(listWatcher, &k6tv1.VirtualMachineInstance{}, 1*time.Hour, cache.Indexers{}) + + vmiInformer.AddEventHandler(cache.ResourceEventHandlerFuncs { + DeleteFunc: handleDeletedVmi, + }) return vmiInformer, nil } + +func handleDeletedVmi(obj interface{}) { + vmi, ok := obj.(*k6tv1.VirtualMachineInstance) + if !ok { + return + } + + // get VMI name in the format namespace/name + vmiKey, err := cache.MetaNamespaceKeyFunc(vmi) + if err != nil { + fmt.Println(err) + return + } + + // check if deleted VMI is in list of VMIs that need to be restarted + _, exists := vmisPendingUpdate[vmiKey] + if !exists { + return + } + + // remove warning label from VM + // if removing the warning label fails, exit before removing VMI from list + // since the label is still there to tell the user to restart, it wouldn't + // make sense to have a mismatch between the number of VMs with the label + // and the number of VMIs in the list of VMIs pending update. + + err = removeWarningLabel(vmi) + if err != nil { + fmt.Println(err) + return + } + + // remove deleted VMI from list + delete(vmisPendingUpdate, vmiKey) + + // check if VMI list is now empty, to signal exiting the job + if len(vmisPendingUpdate) == 0 { + close(exitJob) + } +} + +func removeWarningLabel(vmi *k6tv1.VirtualMachineInstance) error { + virtCli, err := getVirtCli() + if err != nil { + return err + } + + removeLabel := fmt.Sprint(`{"op": "remove", "path": "/metadata/labels/restart-vm-required"}`) + _, err = virtCli.VirtualMachine(vmi.Namespace).Patch(vmi.Name, types.JSONPatchType, []byte(removeLabel), &k8sv1.PatchOptions{}) + return err +} diff --git a/main.go b/main.go index c594111..04ae9d5 100644 --- a/main.go +++ b/main.go @@ -1,16 +1,48 @@ package mass_machine_type_transition -import "os" +import ( + "fmt" + "os" + "strconv" +) func main() { - vmiInformer, err := getVmiInformer() + var err error + // update restartNow if env is set + restartEnv, exists := os.LookupEnv("FORCE_RESTART") + if exists { + restartNow, err = strconv.ParseBool(restartEnv) + if err != nil { + fmt.Println(err) + } + } + + // update namespace if env is set + namespaceEnv, exists := os.LookupEnv("NAMESPACE") + if exists { + namespace = namespaceEnv + } + + virtCli, err := getVirtCli() if err != nil { os.Exit(1) } - - if vmiInformer != nil { + + vmiInformer, err := getVmiInformer(virtCli) + if err != nil { os.Exit(1) } - + + go vmiInformer.Run(exitJob) + + + err = updateMachineTypes(virtCli) + if err != nil { + fmt.Println(err) + } + + // wait for list of VMIs that need restart to be empty + <-exitJob + os.Exit(0) } diff --git a/mass_machine_type_transition_job.yaml b/mass_machine_type_transition_job.yaml new file mode 100644 index 0000000..9997b8e --- /dev/null +++ b/mass_machine_type_transition_job.yaml @@ -0,0 +1,19 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: mass-machine-type-transition +spec: + template: + metadata: + name: mass-machine-type-transition + spec: + containers: + - name: machine-type-transition + image: machine-type-transition:latest + imagePullPolicy: IfNotPresent + env: + - name: NAMESPACE + value: "" + - name: FORCE_RESTART + value: false + restartPolicy: OnFailure diff --git a/update_machine_type.go b/update_machine_type.go new file mode 100644 index 0000000..9ec5d7b --- /dev/null +++ b/update_machine_type.go @@ -0,0 +1,109 @@ +package mass_machine_type_transition + +import ( + "fmt" + "strings" + + k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + k6tv1 "kubevirt.io/api/core/v1" + "kubevirt.io/client-go/kubecli" +) + +// using this as a const allows us to easily modify the program to update if a newer version is released +// we generally want to be updating the machine types to the most recent version +const latestMachineTypeVersion = "rhel9.2.0" + +var ( + vmisPendingUpdate = make(map[string]struct{}) + exitJob = make(chan struct{}) + + // by default, update machine type across all namespaces + namespace = k8sv1.NamespaceAll + //labels []string + + // by default, should require manual restarting of VMIs + restartNow = false +) + +func patchVmMachineType(virtCli kubecli.KubevirtClient, vm *k6tv1.VirtualMachine, machineType string) error { + updateMachineType := fmt.Sprintf(`{"spec": {"template": {"spec": {"domain": {"machine": {"type": "%s"}}}}}}`, machineType) + + _, err := virtCli.VirtualMachine(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(updateMachineType), &k8sv1.PatchOptions{}) + if err != nil { + return err + } + + // add label to running VMs that a restart is required for change to take place + if vm.Status.Created { + // adding the warning label to the VMs regardless if we restart them now or if the user does it manually + // shouldn't matter, since the deletion of the VMI will remove the label and remove the vmi list anyway + err = addWarningLabel(virtCli, vm) + if err != nil { + return err + } + + if restartNow { + err = virtCli.VirtualMachine(vm.Namespace).Restart(vm.Name, &k6tv1.RestartOptions{}) + if err != nil { + return err + } + } + } + + return nil +} + +func updateMachineTypes(virtCli kubecli.KubevirtClient) error { + vmList, err := virtCli.VirtualMachine(namespace).List(&k8sv1.ListOptions{}) + if err != nil { + return err + } + for _, vm := range vmList.Items { + machineType := vm.Spec.Template.Spec.Domain.Machine.Type + machineTypeSubstrings := strings.Split(machineType, "-") + + // in the case where q35 is the machine type, the VMI status will contain the + // full machine type string. Since q35 is an alias for the most recent machine type, + // if a VM hasn't been restarted and thus still has an outdated machine type, it should + // be updated to the most recent machine type. I'm not able to access the machine type from + // VMI status using the kv library, so instead I am opting to update the machine types of all + // VMs with q35 instead. It might only be necessary to update the ones that are running as well, + // though I am not sure. + + if len(machineTypeSubstrings) == 1 { + if machineTypeSubstrings[0] == "q35" { + machineType = "pc-q35-" + latestMachineTypeVersion + return patchVmMachineType(virtCli, &vm, machineType) + } + } + + if len(machineTypeSubstrings) == 3 { + version := machineTypeSubstrings[2] + if strings.Contains(version, "rhel") && version < "rhel9.0.0" { + machineTypeSubstrings[2] = latestMachineTypeVersion + machineType = strings.Join(machineTypeSubstrings, "-") + return patchVmMachineType(virtCli, &vm, machineType) + } + } + } + return nil +} + +func addWarningLabel (virtCli kubecli.KubevirtClient, vm *k6tv1.VirtualMachine) error { + addLabel := fmt.Sprint(`{"metadata": {"labels": {"restart-vm-required": "true"}}}}}`) + _, err := virtCli.VirtualMachine(vm.Namespace).Patch(vm.Name, types.StrategicMergePatchType, []byte(addLabel), &k8sv1.PatchOptions{}) + if err != nil { + return err + } + + // get VM name in the format namespace/name + vmKey, err := cache.MetaNamespaceKeyFunc(vm) + if err != nil { + return err + } + vmisPendingUpdate[vmKey] = struct{}{} + + return nil +}