Skip to content

Commit

Permalink
Refactor v1a2 backup to store VM-related YAML in a single ExtraConfig…
Browse files Browse the repository at this point in the history 
… key
  • Loading branch information
@dilyar85
dilyar85 committed Dec 18, 2023
1 parent 690aff2 commit 02c870d
Show file tree
Hide file tree
Showing 7 changed files with 421 additions and 216 deletions.
16 changes: 7 additions & 9 deletions api/v1alpha2/virtualmachine_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,15 +117,13 @@ const (
ManagedByExtensionKey = "com.vmware.vcenter.wcp"
ManagedByExtensionType = "VirtualMachine"

// VMBackupKubeDataExtraConfigKey is the ExtraConfig key to persist the VM's
// Kubernetes resource spec data, compressed using gzip and base64-encoded.
VMBackupKubeDataExtraConfigKey = "vmservice.virtualmachine.kubedata"
// VMBackupBootstrapDataExtraConfigKey is the ExtraConfig key to persist the
// VM's bootstrap data object, compressed using gzip and base64-encoded.
VMBackupBootstrapDataExtraConfigKey = "vmservice.virtualmachine.bootstrapdata"
// VMBackupDiskDataExtraConfigKey is the ExtraConfig key to persist the VM's
// attached disk info in JSON, compressed using gzip and base64-encoded.
VMBackupDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata"
// VMBackupKubeObjectsYAMLExtraConfigKey is the ExtraConfig key to persist VM
// and its relevant Kubernetes resource YAML separated by "---", compressed
// using gzip and base64-encoded.
VMBackupKubeObjectsYAMLExtraConfigKey = "vmservice.virtualmachine.kube.objects.yaml"
// VMBackupPVCDiskDataExtraConfigKey is the ExtraConfig key to persist the VM's
// PVC disk data in JSON, compressed using gzip and base64-encoded.
VMBackupPVCDiskDataExtraConfigKey = "vmservice.virtualmachine.pvc.disk.data"
// VMBackupCloudInitInstanceIDExtraConfigKey is the ExtraConfig key to persist
// the VM's Cloud-Init instance ID, compressed using gzip and base64-encoded.
VMBackupCloudInitInstanceIDExtraConfigKey = "vmservice.virtualmachine.cloudinit.instanceid"
Expand Down
13 changes: 0 additions & 13 deletions pkg/context/backupvirtualmachine_context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,3 @@ type BackupVirtualMachineContext struct {
func (c *BackupVirtualMachineContext) String() string {
return fmt.Sprintf("Backup %s", c.VMCtx.String())
}

// BackupVirtualMachineContextA2 is the context used for storing backup data of
// VM and its related objects.
type BackupVirtualMachineContextA2 struct {
VMCtx VirtualMachineContextA2
VcVM *object.VirtualMachine
BootstrapData map[string]string
DiskUUIDToPVC map[string]corev1.PersistentVolumeClaim
}

func (c *BackupVirtualMachineContextA2) String() string {
return fmt.Sprintf("Backup %s", c.VMCtx.String())
}
216 changes: 112 additions & 104 deletions pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,31 @@ package virtualmachine

import (
"encoding/json"
"fmt"
"strings"

corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/yaml"

"github.com/vmware/govmomi/object"
"github.com/vmware/govmomi/vim25/mo"
"github.com/vmware/govmomi/vim25/types"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"

vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
"github.com/vmware-tanzu/vm-operator/pkg/context"
"github.com/vmware-tanzu/vm-operator/pkg/util"
)

type VMDiskData struct {
// BackupVirtualMachineOptions contains the options for BackupVirtualMachine.
type BackupVirtualMachineOptions struct {
VMCtx context.VirtualMachineContextA2
VcVM *object.VirtualMachine
KubeObjects []client.Object
DiskUUIDToPVC map[string]corev1.PersistentVolumeClaim
}

// PVCDiskData contains the data of a disk attached to VM backed by a PVC.
type PVCDiskData struct {
// Filename contains the datastore path to the virtual disk.
FileName string
// PVCName is the name of the PVC backed by the virtual disk.
Expand All @@ -28,128 +40,149 @@ type VMDiskData struct {

// BackupVirtualMachine backs up the required data of a VM into its ExtraConfig.
// Currently, the following data is backed up:
// - Kubernetes VirtualMachine object in YAML format (without its .status field).
// - VM bootstrap data in JSON (if provided).
// - VM disk data in JSON (if created and attached by PVCs).
func BackupVirtualMachine(ctx context.BackupVirtualMachineContextA2) error {
// - VM relevant Kubernetes objects in YAML separated by "---".
// - Cloud-Init instance ID (if not already stored in ExtraConfig).
// - PVC disk data in JSON format (if DiskUUIDToPVC is not empty).
func BackupVirtualMachine(opts BackupVirtualMachineOptions) error {
var moVM mo.VirtualMachine
if err := ctx.VcVM.Properties(ctx.VMCtx, ctx.VcVM.Reference(),
if err := opts.VcVM.Properties(opts.VMCtx, opts.VcVM.Reference(),
[]string{"config.extraConfig"}, &moVM); err != nil {
ctx.VMCtx.Logger.Error(err, "Failed to get VM properties for backup")
opts.VMCtx.Logger.Error(err, "Failed to get VM properties for backup")
return err
}
curEcMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig)

var ecToUpdate []types.BaseOptionValue
curEcMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig)
ecToUpdate := []types.BaseOptionValue{}

vmKubeDataBackup, err := getDesiredVMKubeDataForBackup(ctx.VMCtx.VM, curEcMap)
objectsYAML, err := getDesiredKubeObjectsYAMLForBackup(opts.KubeObjects, curEcMap)
if err != nil {
ctx.VMCtx.Logger.Error(err, "Failed to get VM kube data for backup")
opts.VMCtx.Logger.Error(err, "Failed to get kube objects yaml for backup")
return err
}

if vmKubeDataBackup == "" {
ctx.VMCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged")
if objectsYAML == "" {
opts.VMCtx.Logger.Info("Skipping kube objects yaml backup as unchanged")
} else {
ecToUpdate = append(ecToUpdate, &types.OptionValue{
Key: vmopv1.VMBackupKubeDataExtraConfigKey,
Value: vmKubeDataBackup,
Key: vmopv1.VMBackupKubeObjectsYAMLExtraConfigKey,
Value: objectsYAML,
})
}

instanceIDBackup, err := getDesiredCloudInitInstanceIDForBackup(ctx.VMCtx.VM, curEcMap)
instanceID, err := getDesiredCloudInitInstanceIDForBackup(opts.VMCtx.VM, curEcMap)
if err != nil {
ctx.VMCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup")
opts.VMCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup")
return err
}

if instanceIDBackup == "" {
ctx.VMCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored")
if instanceID == "" {
opts.VMCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored")
} else {
ecToUpdate = append(ecToUpdate, &types.OptionValue{
Key: vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey,
Value: instanceIDBackup,
})
}

bootstrapDataBackup, err := getDesiredBootstrapDataForBackup(ctx.BootstrapData, curEcMap)
if err != nil {
ctx.VMCtx.Logger.Error(err, "Failed to get VM bootstrap data for backup")
return err
}

if bootstrapDataBackup == "" {
ctx.VMCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged")
} else {
ecToUpdate = append(ecToUpdate, &types.OptionValue{
Key: vmopv1.VMBackupBootstrapDataExtraConfigKey,
Value: bootstrapDataBackup,
Value: instanceID,
})
}

diskDataBackup, err := getDesiredDiskDataForBackup(ctx, curEcMap)
pvcDiskData, err := getDesiredPVCDiskDataForBackup(opts, curEcMap)
if err != nil {
ctx.VMCtx.Logger.Error(err, "Failed to get VM disk data for backup")
opts.VMCtx.Logger.Error(err, "Failed to get PVC disk data for backup")
return err
}

if diskDataBackup == "" {
ctx.VMCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged")
if pvcDiskData == "" {
opts.VMCtx.Logger.V(4).Info("Skipping PVC disk data backup as unchanged")
} else {
ecToUpdate = append(ecToUpdate, &types.OptionValue{
Key: vmopv1.VMBackupDiskDataExtraConfigKey,
Value: diskDataBackup,
Key: vmopv1.VMBackupPVCDiskDataExtraConfigKey,
Value: pvcDiskData,
})
}

if len(ecToUpdate) != 0 {
ctx.VMCtx.Logger.Info("Updating VM ExtraConfig with backup data")
ctx.VMCtx.Logger.V(4).Info("", "ExtraConfig", ecToUpdate)
if _, err := ctx.VcVM.Reconfigure(ctx.VMCtx, types.VirtualMachineConfigSpec{
opts.VMCtx.Logger.Info("Updating VM ExtraConfig with latest backup data")
opts.VMCtx.Logger.V(4).Info("", "ExtraConfigToUpdate", ecToUpdate)
if _, err := opts.VcVM.Reconfigure(opts.VMCtx, types.VirtualMachineConfigSpec{
ExtraConfig: ecToUpdate,
}); err != nil {
ctx.VMCtx.Logger.Error(err, "Failed to update VM ExtraConfig for backup")
opts.VMCtx.Logger.Error(err, "Failed to update VM ExtraConfig with latest backup data")
return err
}
}

return nil
}

func getDesiredVMKubeDataForBackup(
vm *vmopv1.VirtualMachine,
func getDesiredKubeObjectsYAMLForBackup(
kubeObjects []client.Object,
ecMap map[string]string) (string, error) {
// If the ExtraConfig already contains the latest VM spec, determined by
// 'metadata.generation', return an empty string to skip the backup.
if ecKubeData, ok := ecMap[vmopv1.VMBackupKubeDataExtraConfigKey]; ok {
vmFromBackup, err := constructVMObj(ecKubeData)
if err != nil {
return "", err
}
if vmFromBackup.ObjectMeta.Generation >= vm.ObjectMeta.Generation {
return "", nil
// Check if the VM's ExtraConfig already contains the latest version of all
// objects, determined by each resource UID to resource version mapping.
var isLatestBackup bool
if ecKubeData, ok := ecMap[vmopv1.VMBackupKubeObjectsYAMLExtraConfigKey]; ok {
if resourceToVersion := tryGetResourceVersion(ecKubeData); resourceToVersion != nil {
isLatestBackup = true
for _, curObj := range kubeObjects {
if curObj.GetResourceVersion() != resourceToVersion[string(curObj.GetUID())] {
isLatestBackup = false
break
}
}
}
}

backupVM := vm.DeepCopy()
backupVM.Status = vmopv1.VirtualMachineStatus{}
backupVMYaml, err := yaml.Marshal(backupVM)
if err != nil {
return "", err
// All objects are up-to-date, return an empty string to skip the backup.
if isLatestBackup {
return "", nil
}

return util.EncodeGzipBase64(string(backupVMYaml))
// Backup the latest version of all objects with encoded and gzipped YAML.
// Use "---" as the separator between objects to allow for easy parsing.
marshaledStrs := []string{}
for _, obj := range kubeObjects {
marshaledYaml, err := yaml.Marshal(obj)
if err != nil {
return "", fmt.Errorf("failed to marshal object %s: %v", obj.GetName(), err)
}
marshaledStrs = append(marshaledStrs, string(marshaledYaml))
}

kubeObjectsYAML := strings.Join(marshaledStrs, "---\n")
return util.EncodeGzipBase64(kubeObjectsYAML)
}

func constructVMObj(ecKubeData string) (vmopv1.VirtualMachine, error) {
var vmObj vmopv1.VirtualMachine
decodedKubeData, err := util.TryToDecodeBase64Gzip([]byte(ecKubeData))
func tryGetResourceVersion(ecKubeObjectsYAML string) map[string]string {
decoded, err := util.TryToDecodeBase64Gzip([]byte(ecKubeObjectsYAML))
if err != nil {
return vmObj, err
return nil
}

resourceVersions := map[string]string{}
kubeObjectsYAML := strings.Split(decoded, "---\n")
for _, objYAML := range kubeObjectsYAML {
if objYAML != "" {
objYAML = strings.TrimSpace(objYAML)
var obj map[string]interface{}
if err := yaml.Unmarshal([]byte(objYAML), &obj); err != nil {
continue
}
metadata, ok := obj["metadata"].(map[string]interface{})
if !ok {
continue
}
uid, ok := metadata["uid"].(string)
if !ok {
continue
}
resourceVersion, ok := metadata["resourceVersion"].(string)
if !ok {
continue
}
resourceVersions[uid] = resourceVersion
}
}

err = yaml.Unmarshal([]byte(decodedKubeData), &vmObj)
return vmObj, err
return resourceVersions
}

func getDesiredCloudInitInstanceIDForBackup(
Expand All @@ -169,50 +202,25 @@ func getDesiredCloudInitInstanceIDForBackup(
return util.EncodeGzipBase64(instanceID)
}

func getDesiredBootstrapDataForBackup(
bootstrapDataRaw map[string]string,
ecMap map[string]string) (string, error) {
// No bootstrap data is specified, return an empty string to skip the backup.
if len(bootstrapDataRaw) == 0 {
return "", nil
}

bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw)
if err != nil {
return "", err
}
bootstrapDataBackup, err := util.EncodeGzipBase64(string(bootstrapDataJSON))
if err != nil {
return "", err
}

// Return an empty string to skip the backup if the data is unchanged.
if bootstrapDataBackup == ecMap[vmopv1.VMBackupBootstrapDataExtraConfigKey] {
return "", nil
}

return bootstrapDataBackup, nil
}

func getDesiredDiskDataForBackup(
ctx context.BackupVirtualMachineContextA2,
func getDesiredPVCDiskDataForBackup(
opts BackupVirtualMachineOptions,
ecMap map[string]string) (string, error) {
// Return an empty string to skip backup if no disk uuid to PVC is specified.
if len(ctx.DiskUUIDToPVC) == 0 {
if len(opts.DiskUUIDToPVC) == 0 {
return "", nil
}

deviceList, err := ctx.VcVM.Device(ctx.VMCtx)
deviceList, err := opts.VcVM.Device(opts.VMCtx)
if err != nil {
return "", err
}

var diskData []VMDiskData
var diskData []PVCDiskData
for _, device := range deviceList.SelectByType((*types.VirtualDisk)(nil)) {
if disk, ok := device.(*types.VirtualDisk); ok {
if b, ok := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok {
if pvc, ok := ctx.DiskUUIDToPVC[b.Uuid]; ok {
diskData = append(diskData, VMDiskData{
if pvc, ok := opts.DiskUUIDToPVC[b.Uuid]; ok {
diskData = append(diskData, PVCDiskData{
FileName: b.FileName,
PVCName: pvc.Name,
AccessModes: pvc.Spec.AccessModes,
Expand All @@ -232,7 +240,7 @@ func getDesiredDiskDataForBackup(
}

// Return an empty string to skip the backup if the data is unchanged.
if diskDataBackup == ecMap[vmopv1.VMBackupDiskDataExtraConfigKey] {
if diskDataBackup == ecMap[vmopv1.VMBackupPVCDiskDataExtraConfigKey] {
return "", nil
}

Expand Down
Loading

0 comments on commit 02c870d

Please sign in to comment.