Support Bring Your Own (Encryption) Key (BYOK)

This patch adds support for bringing your own encryption key used
to encrypt/recrypt VMs.

api/v1alpha1/virtualmachine_conversion.go

@@ -822,6 +822,10 @@ func Convert_v1alpha3_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus(
 	return nil
 }
 
+func restore_v1alpha3_VirtualMachineEncryptionClass(dst, src *vmopv1.VirtualMachine) {
+	dst.Spec.EncryptionClassName = src.Spec.EncryptionClassName
+}
+
 func restore_v1alpha3_VirtualMachineImage(dst, src *vmopv1.VirtualMachine) {
 	dst.Spec.Image = src.Spec.Image
 	dst.Spec.ImageName = src.Spec.ImageName
@@ -1239,6 +1243,7 @@ func (src *VirtualMachine) ConvertTo(dstRaw ctrlconversion.Hub) error {
 	restore_v1alpha3_VirtualMachineInstanceUUID(dst, restored)
 	restore_v1alpha3_VirtualMachineGuestID(dst, restored)
 	restore_v1alpha3_VirtualMachineCdrom(dst, restored)
+	restore_v1alpha3_VirtualMachineEncryptionClass(dst, restored)
 
 	// END RESTORE
 

api/v1alpha2/virtualmachine_conversion.go

@@ -142,6 +142,10 @@ func Convert_v1alpha3_VirtualMachine_To_v1alpha2_VirtualMachine(
 	return nil
 }
 
+func restore_v1alpha3_VirtualMachineEncryptionClass(dst, src *vmopv1.VirtualMachine) {
+	dst.Spec.EncryptionClassName = src.Spec.EncryptionClassName
+}
+
 func restore_v1alpha3_VirtualMachineImage(dst, src *vmopv1.VirtualMachine) {
 	dst.Spec.Image = src.Spec.Image
 	dst.Spec.ImageName = src.Spec.ImageName
@@ -293,6 +297,7 @@ func (src *VirtualMachine) ConvertTo(dstRaw ctrlconversion.Hub) error {
 	restore_v1alpha3_VirtualMachineSpecNetworkDomainName(dst, restored)
 	restore_v1alpha3_VirtualMachineGuestID(dst, restored)
 	restore_v1alpha3_VirtualMachineCdrom(dst, restored)
+	restore_v1alpha3_VirtualMachineEncryptionClass(dst, restored)
 
 	// END RESTORE
 

api/v1alpha3/virtualmachine_types.go

@@ -41,6 +41,14 @@ const (
 	// VirtualMachineConditionPlacementReady indicates that the placement decision for the VM is ready.
 	VirtualMachineConditionPlacementReady = "VirtualMachineConditionPlacementReady"
 
+	// VirtualMachineEncryptionClassReady indicates that a referenced
+	// EncryptionClass is ready.
+	VirtualMachineEncryptionClassReady = "VirtualMachineEncryptionClassReady"
+
+	// VirtualMachineEncryptionSynced indicates that the VirtualMachine's
+	// encryption state is synced to the desired encryption state.
+	VirtualMachineEncryptionSynced = "VirtualMachineEncryptionSynced"
+
 	// VirtualMachineConditionCreated indicates that the VM has been created.
 	VirtualMachineConditionCreated = "VirtualMachineCreated"
 
@@ -438,6 +446,27 @@ type VirtualMachineSpec struct {
 
 	// +optional
 
+	// EncryptionClassName describes the name of the EncryptionClass resource
+	// used to encrypt this VM.
+	//
+	// Please note, this field is not required to encrypt the VM. If the
+	// underlying platform implements the vSphere Native Key Provider (NKP) then
+	// the VM may still be fully or partially encrypted depending on the
+	// specified storage and VM classes.
+	//
+	// If an NKP is present and an encryption storage class is selected, the
+	// VM's home files and non-PVC disks will be encrypted.
+	//
+	// If an NKP is present and a VM Class with a virtual, trusted platform
+	// module (vTPM) is selected, the VM's home files will be encrypted.
+	//
+	// If the underlying vSphere platform does not have an NKP, then this field
+	// is required when specifying an encryption storage class and/or a VM Class
+	// with a vTPM.
+	EncryptionClassName string `json:"encryptionClassName,omitempty"`
+
+	// +optional
+
 	// StorageClass describes the name of a Kubernetes StorageClass resource
 	// used to configure this VM's storage-related attributes.
 	//

config/crd/bases/vmoperator.vmware.com_virtualmachinereplicasets.yaml

@@ -1006,6 +1006,30 @@ spec:
                           an existing VM on the underlying platform that was not deployed from a
                           VM class.
                         type: string
+                      encryptionClassName:
+                        description: |-
+                          EncryptionClassName describes the name of the EncryptionClass resource
+                          used to encrypt this VM.
+
+
+                          Please note, this field is not required to encrypt the VM. If the
+                          underlying platform implements the vSphere Native Key Provider (NKP) then
+                          the VM may still be fully or partially encrypted depending on the
+                          specified storage and VM classes.
+
+
+                          If an NKP is present and an encryption storage class is selected, the
+                          VM's home files and non-PVC disks will be encrypted.
+
+
+                          If an NKP is present and a VM Class with a virtual, trusted platform
+                          module (vTPM) is selected, the VM's home files will be encrypted.
+
+
+                          If the underlying vSphere platform does not have an NKP, then this field
+                          is required when specifying an encryption storage class and/or a VM Class
+                          with a vTPM.
+                        type: string
                       guestID:
                         description: |-
                           GuestID describes the desired guest operating system identifier for a VM.

config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml

@@ -3790,6 +3790,30 @@ spec:
                   an existing VM on the underlying platform that was not deployed from a
                   VM class.
                 type: string
+              encryptionClassName:
+                description: |-
+                  EncryptionClassName describes the name of the EncryptionClass resource
+                  used to encrypt this VM.
+
+
+                  Please note, this field is not required to encrypt the VM. If the
+                  underlying platform implements the vSphere Native Key Provider (NKP) then
+                  the VM may still be fully or partially encrypted depending on the
+                  specified storage and VM classes.
+
+
+                  If an NKP is present and an encryption storage class is selected, the
+                  VM's home files and non-PVC disks will be encrypted.
+
+
+                  If an NKP is present and a VM Class with a virtual, trusted platform
+                  module (vTPM) is selected, the VM's home files will be encrypted.
+
+
+                  If the underlying vSphere platform does not have an NKP, then this field
+                  is required when specifying an encryption storage class and/or a VM Class
+                  with a vTPM.
+                type: string
               guestID:
                 description: |-
                   GuestID describes the desired guest operating system identifier for a VM.

controllers/virtualmachine/virtualmachine/virtualmachine_controller.go

@@ -24,6 +24,7 @@ import (
 
 	vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha3"
 
+	byokv1 "github.com/vmware-tanzu/vm-operator/external/byok/api/v1alpha1"
 	"github.com/vmware-tanzu/vm-operator/pkg/conditions"
 	pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config"
 	pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context"
@@ -104,6 +105,14 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err
 	builder = builder.Watches(&vmopv1.VirtualMachineClass{},
 		handler.EnqueueRequestsFromMapFunc(classToVMMapperFn(ctx, r.Client, isDefaultVMClassController)))
 
+	if pkgcfg.FromContext(ctx).Features.BringYourOwnEncryptionKey {
+		builder = builder.Watches(
+			&byokv1.EncryptionClass{},
+			handler.EnqueueRequestsFromMapFunc(
+				encryptionClassToVMMapperFn(ctx, r.Client),
+			))
+	}
+
 	return builder.Complete(r)
 }
 
@@ -167,6 +176,54 @@ func classToVMMapperFn(
 	}
 }
 
+// encryptionClassToVMMapperFn returns a mapper function that can be used to
+// enqueue reconcile requests for VMs in response to an event on the
+// EncryptionClass resource.
+func encryptionClassToVMMapperFn(
+	ctx *pkgctx.ControllerManagerContext,
+	c client.Client) func(_ context.Context, o client.Object) []reconcile.Request {
+
+	// For a given EncryptionClass, return reconcile requests for VMs that
+	// specify the same EncryptionClass.
+	return func(_ context.Context, o client.Object) []reconcile.Request {
+		obj := o.(*byokv1.EncryptionClass)
+		logger := ctx.Logger.WithValues("name", obj.Name, "namespace", obj.Namespace)
+
+		logger.V(4).Info("Reconciling all VMs referencing an EncryptionClass")
+
+		// Find all VM resources that reference this EncryptionClass.
+		vmList := &vmopv1.VirtualMachineList{}
+		if err := c.List(ctx, vmList, client.InNamespace(obj.Namespace)); err != nil {
+			logger.Error(
+				err,
+				"Failed to list VirtualMachines for reconciliation due to EncryptionClass watch")
+			return nil
+		}
+
+		// Populate reconcile requests for VMs that reference this
+		// EncryptionClass.
+		var requests []reconcile.Request
+		for _, vm := range vmList.Items {
+			if vm.Spec.EncryptionClassName == obj.Name {
+				requests = append(
+					requests,
+					reconcile.Request{
+						NamespacedName: client.ObjectKey{
+							Namespace: vm.Namespace,
+							Name:      vm.Name,
+						},
+					})
+			}
+		}
+
+		logger.Info(
+			"Returning VM reconcile requests due to EncryptionClass watch",
+			"requests", requests)
+
+		return requests
+	}
+}
+
 func upgradeSchema(ctx *pkgctx.VirtualMachineContext) {
 	// If empty, this VM was created before v1alpha3 added the spec.instanceUUID field.
 	if ctx.VM.Spec.InstanceUUID == "" && ctx.VM.Status.InstanceUUID != "" {

pkg/providers/vsphere/constants/constants.go

@@ -67,6 +67,10 @@ const (
 	CloudInitGuestInfoUserdata         = "guestinfo.userdata"
 	CloudInitGuestInfoUserdataEncoding = "guestinfo.userdata.encoding"
 
+	// CryptoIDAnnotation is the annotation key used to store the MD5 hash of
+	// the public provider and key IDs used to encrypt or recrypt a VM.
+	CryptoIDAnnotation = pkg.VMOperatorKey + "/crypto-provider-and-key-id-hash"
+
 	// InstanceStoragePVCNamePrefix prefix of auto-generated PVC names.
 	InstanceStoragePVCNamePrefix = "instance-pvc-"
 	// InstanceStorageLabelKey identifies resources related to instance storage.

pkg/providers/vsphere/vmprovider_vm.go

@@ -13,6 +13,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/go-logr/logr"
 	"github.com/vmware/govmomi/object"
 	"github.com/vmware/govmomi/pbm"
 	"github.com/vmware/govmomi/pbm/types"
@@ -515,7 +516,8 @@ func (vs *vSphereVMProvider) updateVirtualMachine(
 		}
 
 		getUpdateArgsFn := func() (*vmUpdateArgs, error) {
-			// TODO: Use createArgs if we already got them
+			// TODO: Use createArgs if we already got them, except for:
+			//       - createArgs.ConfigSpec.Crypto
 			_ = createArgs
 			return vs.vmUpdateGetArgs(vmCtx)
 		}
@@ -1076,6 +1078,13 @@ func (vs *vSphereVMProvider) vmCreateGenConfigSpec(
 		createArgs.ImageStatus,
 		minCPUFreq)
 
+	// Get the encryption class details for the VM.
+	if pkgcfg.FromContext(vmCtx).Features.BringYourOwnEncryptionKey {
+		if err := vs.vmGenConfigSpecCryptSpec(vmCtx, &createArgs.ConfigSpec); err != nil {
+			return err
+		}
+	}
+
 	err := vs.vmCreateGenConfigSpecExtraConfig(vmCtx, createArgs)
 	if err != nil {
 		return err
@@ -1094,6 +1103,18 @@ func (vs *vSphereVMProvider) vmCreateGenConfigSpec(
 	return nil
 }
 
+func (vs *vSphereVMProvider) vmGenConfigSpecCryptSpec(
+	vmCtx pkgctx.VirtualMachineContext,
+	configSpec *vimtypes.VirtualMachineConfigSpec) error {
+
+	return VMGenConfigSpecCryptoSpec(
+		logr.NewContext(vmCtx, vmCtx.Logger),
+		vs.k8sClient,
+		vmCtx.VM,
+		vmCtx.MoVM,
+		configSpec)
+}
+
 func (vs *vSphereVMProvider) vmCreateGenConfigSpecExtraConfig(
 	vmCtx pkgctx.VirtualMachineContext,
 	createArgs *VMCreateArgs) error {
@@ -1279,6 +1300,13 @@ func (vs *vSphereVMProvider) vmUpdateGetArgs(
 		vmopv1.VirtualMachineImageStatus{},
 		updateArgs.MinCPUFreq)
 
+	// Get the encryption class details for the VM.
+	if pkgcfg.FromContext(vmCtx).Features.BringYourOwnEncryptionKey {
+		if err := vs.vmGenConfigSpecCryptSpec(vmCtx, &updateArgs.ConfigSpec); err != nil {
+			return nil, err
+		}
+	}
+
 	return updateArgs, nil
 }
 
@@ -1326,5 +1354,12 @@ func (vs *vSphereVMProvider) vmResizeGetArgs(
 			minCPUFreq)
 	}
 
+	// Get the encryption class details for the VM.
+	if pkgcfg.FromContext(vmCtx).Features.BringYourOwnEncryptionKey {
+		if err := vs.vmGenConfigSpecCryptSpec(vmCtx, &resizeArgs.ConfigSpec); err != nil {
+			return nil, err
+		}
+	}
+
 	return resizeArgs, nil
 }