diff --git a/api/v1alpha1/condition_conversion.go b/api/v1alpha1/condition_conversion.go index 3211b0c24..275ef4410 100644 --- a/api/v1alpha1/condition_conversion.go +++ b/api/v1alpha1/condition_conversion.go @@ -17,6 +17,12 @@ func Convert_v1alpha1_Condition_To_v1_Condition(in *Condition, out *metav1.Condi out.Reason = in.Reason out.Message = in.Message + // The metav1.Condition requires the reason to be non-empty, when it was not in our prior v1a1 Condition. + // We don't have any great options as to what we can fill this in as. + if out.Reason == "" { + out.Reason = string(out.Status) + } + // TODO: out.ObservedGeneration = return nil diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index 905fb8405..536695bd3 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -155,7 +155,7 @@ func convert_v1alpha1_VmMetadata_To_v1alpha2_BootstrapSpec( out.CloudInit = &v1alpha2.VirtualMachineBootstrapCloudInitSpec{ RawCloudConfig: corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: objectName}, - Key: "guestinfo.userdata", + Key: "guestinfo.userdata", // TODO: Is this good enough? v1a1 would include everything with the "guestinfo" prefix }, } case VirtualMachineMetadataOvfEnvTransport: @@ -449,6 +449,12 @@ func convert_v1alpha2_NetworkStatus_To_v1alpha1_Network( func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( in *VirtualMachineSpec, out *v1alpha2.VirtualMachineSpec, s apiconversion.Scope) error { + // The generated auto convert will convert the power modes as-is strings which breaks things, so keep + // this first. + if err := autoConvert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha1_VirtualMachinePowerState_To_v1alpha2_VirtualMachinePowerState(in.PowerState) out.PowerOffMode = convert_v1alpha1_VirtualMachinePowerOpMode_To_v1alpha2_VirtualMachinePowerOpMode(in.PowerOffMode) out.SuspendMode = convert_v1alpha1_VirtualMachinePowerOpMode_To_v1alpha2_VirtualMachinePowerOpMode(in.SuspendMode) @@ -469,12 +475,16 @@ func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( // Deprecated: // in.Ports - return autoConvert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec(in, out, s) + return nil } func Convert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec( in *v1alpha2.VirtualMachineSpec, out *VirtualMachineSpec, s apiconversion.Scope) error { + if err := autoConvert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha2_VirtualMachinePowerState_To_v1alpha1_VirtualMachinePowerState(in.PowerState) out.PowerOffMode = convert_v1alpha2_VirtualMachinePowerOpMode_To_v1alpha1_VirtualMachinePowerOpMode(in.PowerOffMode) out.SuspendMode = convert_v1alpha2_VirtualMachinePowerOpMode_To_v1alpha1_VirtualMachinePowerOpMode(in.SuspendMode) @@ -496,7 +506,7 @@ func Convert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec( // Deprecated: // out.Ports - return autoConvert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec(in, out, s) + return nil } func Convert_v1alpha1_VirtualMachineVolumeStatus_To_v1alpha2_VirtualMachineVolumeStatus( @@ -518,18 +528,26 @@ func Convert_v1alpha2_VirtualMachineVolumeStatus_To_v1alpha1_VirtualMachineVolum func Convert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus( in *VirtualMachineStatus, out *v1alpha2.VirtualMachineStatus, s apiconversion.Scope) error { + if err := autoConvert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha1_VirtualMachinePowerState_To_v1alpha2_VirtualMachinePowerState(in.PowerState) out.Network = convert_v1alpha1_Network_To_v1alpha2_NetworkStatus(in.VmIp, in.NetworkInterfaces) out.LastRestartTime = in.LastRestartTime // WARNING: in.Phase requires manual conversion: does not exist in peer-type - return autoConvert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus(in, out, s) + return nil } func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( in *v1alpha2.VirtualMachineStatus, out *VirtualMachineStatus, s apiconversion.Scope) error { + if err := autoConvert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha2_VirtualMachinePowerState_To_v1alpha1_VirtualMachinePowerState(in.PowerState) out.Phase = convert_v1alpha2_Conditions_To_v1alpha1_Phase(in.Conditions) out.VmIp, out.NetworkInterfaces = convert_v1alpha2_NetworkStatus_To_v1alpha1_Network(in.Network) @@ -538,7 +556,7 @@ func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( // WARNING: in.Image requires manual conversion: does not exist in peer-type // WARNING: in.Class requires manual conversion: does not exist in peer-type - return autoConvert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus(in, out, s) + return nil } // ConvertTo converts this VirtualMachine to the Hub version. diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index faaceb654..8b124ee10 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -103,19 +103,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachine failurePolicy: Fail - name: default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - virtualmachineclasses + - virtualmachines sideEffects: None - admissionReviewVersions: - v1 @@ -124,9 +124,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass failurePolicy: Fail - name: default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -136,7 +136,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachinepublishrequests + - virtualmachineclasses sideEffects: None - admissionReviewVersions: - v1 @@ -145,19 +145,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineclass failurePolicy: Fail - name: default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachineclass.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - virtualmachineservices + - virtualmachineclasses sideEffects: None - admissionReviewVersions: - v1 @@ -166,9 +166,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest failurePolicy: Fail - name: default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -178,7 +178,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachinesetresourcepolicies + - virtualmachinepublishrequests sideEffects: None - admissionReviewVersions: - v1 @@ -187,19 +187,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinepublishrequest failurePolicy: Fail - name: default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachinepublishrequest.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - webconsolerequests + - virtualmachinepublishrequests sideEffects: None - admissionReviewVersions: - v1 @@ -208,19 +208,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachine + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice failurePolicy: Fail - name: default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachines + - virtualmachineservices sideEffects: None - admissionReviewVersions: - v1 @@ -229,9 +229,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineclass + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineservice failurePolicy: Fail - name: default.validating.virtualmachineclass.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachineservice.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -241,7 +241,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachineclasses + - virtualmachineservices sideEffects: None - admissionReviewVersions: - v1 @@ -250,19 +250,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinepublishrequest + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy failurePolicy: Fail - name: default.validating.virtualmachinepublishrequest.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachinepublishrequests + - virtualmachinesetresourcepolicies sideEffects: None - admissionReviewVersions: - v1 @@ -271,9 +271,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineservice + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinesetresourcepolicy failurePolicy: Fail - name: default.validating.virtualmachineservice.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachinesetresourcepolicy.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -283,7 +283,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachineservices + - virtualmachinesetresourcepolicies sideEffects: None - admissionReviewVersions: - v1 @@ -292,17 +292,17 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinesetresourcepolicy + path: /default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest failurePolicy: Fail - name: default.validating.virtualmachinesetresourcepolicy.v1alpha2.vmoperator.vmware.com + name: default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachinesetresourcepolicies + - webconsolerequests sideEffects: None diff --git a/main.go b/main.go index c40785abc..65958e50e 100644 --- a/main.go +++ b/main.go @@ -18,9 +18,12 @@ import ( klog "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/controllers" "github.com/vmware-tanzu/vm-operator/pkg" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/manager" "github.com/vmware-tanzu/vm-operator/webhooks" @@ -265,6 +268,12 @@ func main() { return err } + if lib.IsVMServiceV1Alpha2FSSEnabled() { + if err := addConversionWebhooksToManager(ctx, mgr); err != nil { + return err + } + } + return webhooks.AddToManager(ctx, mgr) } @@ -296,6 +305,57 @@ func main() { } } +// addConversionWebhooksToManager adds the ctrl-runtime managed webhooks. We just use these +// for version conversion, but they can also do mutation and validation webhook callbacks +// instead of our separate webhooks. +func addConversionWebhooksToManager(_ *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if err := (&v1alpha1.VirtualMachine{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineClass{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.ClusterVirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachinePublishRequest{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineService{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineSetResourcePolicy{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + + if err := (&v1alpha2.VirtualMachine{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineClass{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.ClusterVirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachinePublishRequest{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineService{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineSetResourcePolicy{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + + return nil +} + func configureWebhookTLS(opts *webhook.Options) { tlsCfgFunc := func(cfg *tls.Config) { cfg.MinVersion = tls.VersionTLS12 diff --git a/pkg/conditions2/setter.go b/pkg/conditions2/setter.go index b6f6520dc..0194fe577 100644 --- a/pkg/conditions2/setter.go +++ b/pkg/conditions2/setter.go @@ -78,13 +78,20 @@ func Set(to Setter, condition *metav1.Condition) { func TrueCondition(t string) *metav1.Condition { return &metav1.Condition{ Type: t, - Reason: t, // BMV: This is required field in metav1.Conditions. Fixup API later. Status: metav1.ConditionTrue, + // This is a non-empty field in metav1.Conditions, when it was not in our v1a1 Conditions. This + // really doesn't work with how we've defined our conditions so do something to make things + // work for now. + Reason: string(metav1.ConditionTrue), } } // FalseCondition returns a condition with Status=False and the given type. func FalseCondition(t string, reason string, messageFormat string, messageArgs ...interface{}) *metav1.Condition { + if reason == "" { + reason = string(metav1.ConditionFalse) + } + return &metav1.Condition{ Type: t, Status: metav1.ConditionFalse, @@ -95,6 +102,10 @@ func FalseCondition(t string, reason string, messageFormat string, messageArgs . // UnknownCondition returns a condition with Status=Unknown and the given type. func UnknownCondition(t string, reason string, messageFormat string, messageArgs ...interface{}) *metav1.Condition { + if reason == "" { + reason = string(metav1.ConditionUnknown) + } + return &metav1.Condition{ Type: t, Status: metav1.ConditionUnknown, @@ -124,14 +135,14 @@ func SetSummary(to Setter, options ...MergeOption) { Set(to, summary(to, options...)) } -// SetMirror creates a new condition by mirroring the the Ready condition from a dependent object; -// if the Ready condition does not exists in the source object, no target conditions is generated. +// SetMirror creates a new condition by mirroring the Ready condition from a dependent object; +// if the Ready condition does not exist in the source object, no target conditions is generated. func SetMirror(to Setter, targetCondition string, from Getter, options ...MirrorOptions) { Set(to, mirror(from, targetCondition, options...)) } -// SetAggregate creates a new condition with the aggregation of all the the Ready condition -// from a list of dependent objects; if the Ready condition does not exists in one of the source object, +// SetAggregate creates a new condition with the aggregation of all the Ready condition +// from a list of dependent objects; if the Ready condition does not exist in one of the source object, // the object is excluded from the aggregation; if none of the source object have ready condition, // no target conditions is generated. func SetAggregate(to Setter, targetCondition string, from []Getter, options ...MergeOption) { diff --git a/pkg/conditions2/setter_test.go b/pkg/conditions2/setter_test.go index ceb78d52a..5d9e47b25 100644 --- a/pkg/conditions2/setter_test.go +++ b/pkg/conditions2/setter_test.go @@ -199,7 +199,7 @@ func TestMarkMethods(t *testing.T) { g.Expect(Get(vm, "conditionFoo")).To(haveSameStateOf(&metav1.Condition{ Type: "conditionFoo", Status: metav1.ConditionTrue, - Reason: "conditionFoo", + Reason: "True", })) // test MarkFalse diff --git a/pkg/lib/env.go b/pkg/lib/env.go index 06591194e..68059b2c0 100644 --- a/pkg/lib/env.go +++ b/pkg/lib/env.go @@ -85,7 +85,11 @@ func GetVMOpNamespaceFromEnv() (string, error) { } var IsNamedNetworkProviderEnabled = func() bool { - return os.Getenv(NetworkProviderType) == NetworkProviderTypeNamed + return GetNetworkProviderType() == NetworkProviderTypeNamed +} + +func GetNetworkProviderType() string { + return os.Getenv(NetworkProviderType) } var IsWcpFaultDomainsFSSEnabled = func() bool { diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index c1d05aca6..374763b4a 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -29,6 +29,7 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/record" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere" + vsphere2 "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" ) // Manager is a VM Operator controller manager. @@ -116,7 +117,12 @@ func New(opts Options) (Manager, error) { func InitializeProviders(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { vmProviderName := fmt.Sprintf("%s/%s/vmProvider", ctx.Namespace, ctx.Name) recorder := record.New(mgr.GetEventRecorderFor(vmProviderName)) - ctx.VMProvider = vsphere.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + + if lib.IsVMServiceV1Alpha2FSSEnabled() { + ctx.VMProviderA2 = vsphere2.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + } else { + ctx.VMProvider = vsphere.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + } return nil } diff --git a/pkg/util/configspec.go b/pkg/util/configspec.go index 3d070bad6..97d07d444 100644 --- a/pkg/util/configspec.go +++ b/pkg/util/configspec.go @@ -165,3 +165,26 @@ func RemoveDevicesFromConfigSpec(configSpec *vimTypes.VirtualMachineConfigSpec, } configSpec.DeviceChange = targetDevChanges } + +// AppendNewExtraConfigValues add the new extra config values if not already present in the extra config. +func AppendNewExtraConfigValues( + extraConfig []vimTypes.BaseOptionValue, + newECMap map[string]string) []vimTypes.BaseOptionValue { + + ecMap := make(map[string]vimTypes.AnyType) + for _, opt := range extraConfig { + if optValue := opt.GetOptionValue(); optValue != nil { + ecMap[optValue.Key] = optValue.Value + } + } + + // Only add fields that aren't already in the ExtraConfig. + var newExtraConfig []vimTypes.BaseOptionValue + for k, v := range newECMap { + if _, exists := ecMap[k]; !exists { + newExtraConfig = append(newExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + + return append(extraConfig, newExtraConfig...) +} diff --git a/pkg/util/configspec_test.go b/pkg/util/configspec_test.go index b5cdf2d20..f53594152 100644 --- a/pkg/util/configspec_test.go +++ b/pkg/util/configspec_test.go @@ -214,6 +214,28 @@ var _ = Describe("RemoveDevicesFromConfigSpec", func() { }) }) +var _ = Describe("AppendNewExtraConfigValues", func() { + + It("only adds new values not already in the ExtraConfig", func() { + ec := []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{ + Key: "key1", + Value: "keep-me", + }, + } + + newECMap := map[string]string{ + "key1": "should-be-ignored", + "key2": "add-me", + } + + newExtraConfig := util.AppendNewExtraConfigValues(ec, newECMap) + Expect(newExtraConfig).To(HaveLen(2)) + Expect(newExtraConfig).To(ContainElement(&vimTypes.OptionValue{Key: "key1", Value: "keep-me"})) + Expect(newExtraConfig).To(ContainElement(&vimTypes.OptionValue{Key: "key2", Value: "add-me"})) + }) +}) + var _ = Describe("SanitizeVMClassConfigSpec", func() { oldVMClassAsConfigFSSEnabledFunc := lib.IsVMClassAsConfigFSSEnabled var ( diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go index 899ff5015..0c3d3aa2b 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go @@ -146,6 +146,7 @@ func vmTests() { ) BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed testConfig.WithVMClassAsConfigDaynDate = true ethCard = types.VirtualEthernetCard{ @@ -1590,6 +1591,8 @@ func vmTests() { Context("Multiple NICs are specified", func() { BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + vm.Spec.NetworkInterfaces = []vmopv1.VirtualMachineNetworkInterface{ { NetworkName: "VM Network", diff --git a/pkg/vmprovider/providers/vsphere2/client/client.go b/pkg/vmprovider/providers/vsphere2/client/client.go new file mode 100644 index 000000000..7477fd621 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client.go @@ -0,0 +1,255 @@ +// Copyright (c) 2018-2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "context" + "net" + "net/url" + "time" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/session/keepalive" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/methods" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +var log = logf.Log.WithName("vsphere").WithName("client") + +type Client struct { + vimClient *vim25.Client + finder *find.Finder + datacenter *object.Datacenter + restClient *rest.Client + contentLibClient contentlibrary.Provider + clusterModClient clustermodules.Provider + sessionManager *session.Manager + config *config.VSphereVMProviderConfig +} + +// Idle time before a keepalive will be invoked. +const keepAliveIdleTime = 5 * time.Minute + +// SoapKeepAliveHandlerFn returns a keepalive handler function suitable for use with the SOAP handler. +// In case the connectivity to VC is down long enough, the session expires. Further attempts to use the +// client yield NotAuthenticated fault. This handler ensures that we re-login the client in those scenarios. +func SoapKeepAliveHandlerFn(sc *soap.Client, sm *session.Manager, userInfo *url.Userinfo) func() error { + return func() error { + ctx := context.Background() + if _, err := methods.GetCurrentTime(ctx, sc); err != nil && isNotAuthenticatedError(err) { + log.Info("Re-authenticating vim client") + if err = sm.Login(ctx, userInfo); err != nil { + if isInvalidLogin(err) { + log.Error(err, "Invalid login in keepalive handler", "url", sc.URL()) + return err + } + } + } else if err != nil { + log.Error(err, "Error in vim25 client's keepalive handler", "url", sc.URL()) + } + + return nil + } +} + +// RestKeepAliveHandlerFn returns a keepalive handler function suitable for use with the REST handler. +// Similar to the SOAP handler, we customize the handler here so we can re-login the client in case the +// REST session expires due to connectivity issues. +func RestKeepAliveHandlerFn(c *rest.Client, userInfo *url.Userinfo) func() error { + return func() error { + ctx := context.Background() + if sess, err := c.Session(ctx); err == nil && sess == nil { + // session is Unauthorized. + log.Info("Re-authenticating REST client") + if err = c.Login(ctx, userInfo); err != nil { + log.Error(err, "Invalid login in keepalive handler", "url", c.URL()) + return err + } + } else if err != nil { + log.Error(err, "Error in rest client's keepalive handler", "url", c.URL()) + } + + return nil + } +} + +// newRestClient creates a rest client which is configured to use a custom keepalive handler function. +func newRestClient(ctx context.Context, vimClient *vim25.Client, config *config.VSphereVMProviderConfig) (*rest.Client, error) { + log.Info("Creating new REST Client", "VcPNID", config.VcPNID, "VcPort", config.VcPort) + restClient := rest.NewClient(vimClient) + + userInfo := url.UserPassword(config.VcCreds.Username, config.VcCreds.Password) + + // Set a custom keepalive handler function + restClient.Transport = keepalive.NewHandlerREST(restClient, keepAliveIdleTime, RestKeepAliveHandlerFn(restClient, userInfo)) + + // Initial login. This will also start the keepalive. + if err := restClient.Login(ctx, userInfo); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "login failed for url: %v", vimClient.URL()) + } + + return restClient, nil +} + +// NewVimClient creates a new vim25 client which is configured to use a custom keepalive handler function. +// Making this public to allow access from other packages when only VimClient is needed. +func NewVimClient(ctx context.Context, config *config.VSphereVMProviderConfig) (*vim25.Client, *session.Manager, error) { + log.Info("Creating new vim Client", "VcPNID", config.VcPNID, "VcPort", config.VcPort) + soapURL, err := soap.ParseURL(net.JoinHostPort(config.VcPNID, config.VcPort)) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse %s:%s", config.VcPNID, config.VcPort) + } + + soapClient := soap.NewClient(soapURL, config.InsecureSkipTLSVerify) + if config.CAFilePath != "" { + err = soapClient.SetRootCAs(config.CAFilePath) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to set root CA %s", config.CAFilePath) + } + } + + vimClient, err := vim25.NewClient(ctx, soapClient) + if err != nil { + return nil, nil, errors.Wrapf(err, "error creating a new vim client for url: %v", soapURL) + } + + if err := vimClient.UseServiceVersion(); err != nil { + return nil, nil, errors.Wrapf(err, "error setting vim client version for url: %v", soapURL) + } + + userInfo := url.UserPassword(config.VcCreds.Username, config.VcCreds.Password) + sm := session.NewManager(vimClient) + + // Set a custom keepalive handler function + vimClient.RoundTripper = keepalive.NewHandlerSOAP(soapClient, keepAliveIdleTime, SoapKeepAliveHandlerFn(soapClient, sm, userInfo)) + + // Initial login. This will also start the keepalive. + if err = sm.Login(ctx, userInfo); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, nil, errors.Wrapf(err, "login failed for url: %v", soapURL) + } + + return vimClient, sm, err +} + +func newFinder( + ctx context.Context, + vimClient *vim25.Client, + config *config.VSphereVMProviderConfig) (*find.Finder, *object.Datacenter, error) { + + finder := find.NewFinder(vimClient, false) + + dcRef, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "Datacenter", Value: config.Datacenter}) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to find Datacenter %q", config.Datacenter) + } + + dc := dcRef.(*object.Datacenter) + finder.SetDatacenter(dc) + + return finder, dc, nil +} + +// NewClient creates a new Client. As a side effect, it creates a vim25 client and a REST client. +func NewClient(ctx context.Context, config *config.VSphereVMProviderConfig) (*Client, error) { + vimClient, sm, err := NewVimClient(ctx, config) + if err != nil { + return nil, err + } + + finder, datacenter, err := newFinder(ctx, vimClient, config) + if err != nil { + return nil, err + } + + restClient, err := newRestClient(ctx, vimClient, config) + if err != nil { + return nil, err + } + + return &Client{ + vimClient: vimClient, + finder: finder, + datacenter: datacenter, + restClient: restClient, + contentLibClient: contentlibrary.NewProvider(restClient), + clusterModClient: clustermodules.NewProvider(restClient), + sessionManager: sm, + config: config, + }, nil +} + +func isNotAuthenticatedError(err error) bool { + if soap.IsSoapFault(err) { + vimFault := soap.ToSoapFault(err).VimFault() + if _, ok := vimFault.(types.NotAuthenticated); ok { + return true + } + } + + return false +} + +func isInvalidLogin(err error) bool { + if soap.IsSoapFault(err) { + vimFault := soap.ToSoapFault(err).VimFault() + if _, ok := vimFault.(types.InvalidLogin); ok { + return true + } + } + + return false +} + +func (c *Client) VimClient() *vim25.Client { + return c.vimClient +} + +func (c *Client) Finder() *find.Finder { + return c.finder +} + +func (c *Client) Datacenter() *object.Datacenter { + return c.datacenter +} + +func (c *Client) RestClient() *rest.Client { + return c.restClient +} + +func (c *Client) ContentLibClient() contentlibrary.Provider { + return c.contentLibClient +} + +func (c *Client) ClusterModuleClient() clustermodules.Provider { + return c.clusterModClient +} + +func (c *Client) Config() *config.VSphereVMProviderConfig { + return c.config +} + +func (c *Client) Logout(ctx context.Context) { + clientURL := c.vimClient.URL() + log.Info("vsphere client logging out from", "VC", clientURL.Host) + if err := c.sessionManager.Logout(ctx); err != nil { + log.Error(err, "Error logging out the vim25 session", "username", clientURL.User.Username(), "host", clientURL.Host) + } + + if err := c.restClient.Logout(ctx); err != nil { + log.Error(err, "Error logging out the rest session", "username", clientURL.User.Username(), "host", clientURL.Host) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go b/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go new file mode 100644 index 000000000..d77790a4f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go @@ -0,0 +1,46 @@ +//go:build !race + +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +var ( + model *simulator.Model + server *simulator.Server + ctx context.Context + tlsTestModel *simulator.Model + tlsServer *simulator.Server + tlsServerCertPath string + tlsServerKeyPath string +) + +var _ = BeforeSuite(func() { + ctx, model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer = test.BeforeSuite() +}) + +var _ = AfterSuite(func() { + test.AfterSuite( + ctx, + model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer) +}) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Client Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/client/client_test.go b/pkg/vmprovider/providers/vsphere2/client/client_test.go new file mode 100644 index 000000000..a5e3aec21 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client_test.go @@ -0,0 +1,366 @@ +//go:build !race + +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client_test + +import ( + "context" + "net/url" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/session/keepalive" + "github.com/vmware/govmomi/simulator" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + + . "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +func testConfig(vcpnid string, vcport string, user string, pass string) *config.VSphereVMProviderConfig { + providerConfig := &config.VSphereVMProviderConfig{ + VcPNID: vcpnid, + VcPort: vcport, + Datacenter: "datacenter-2", + VcCreds: &credentials.VSphereVMProviderCredentials{ + Username: user, + Password: pass, + }, + // Let the tests run without TLS validation by default. + InsecureSkipTLSVerify: true, + } + return providerConfig +} + +var _ = Describe("keepalive handler", func() { + + var ( + sessionIdleTimeout = time.Second / 2 + keepAliveIdle = sessionIdleTimeout / 2 + sessionCheckPause = 3 * sessionIdleTimeout // Avoid unit test thread waking up before session expires + simulatorIdleTime time.Duration + ) + + assertSoapSessionValid := func(ctx context.Context, m *session.Manager) { + s, err := m.UserSession(ctx) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, s).NotTo(BeNil()) + } + + assertSoapSessionExpired := func(ctx context.Context, m *session.Manager) { + s, err := m.UserSession(ctx) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, s).To(BeNil()) + } + + BeforeEach(func() { + // Sharing this variable causes the race detector to trip. + simulator.SessionIdleTimeout = sessionIdleTimeout + }) + + AfterEach(func() { + simulator.SessionIdleTimeout = simulatorIdleTime + }) + + Context("for SOAP sessions", func() { + Context("with a logged out session)", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + m := session.NewManager(c) + + // Session should be valid since simulator logs in the client by default + assertSoapSessionValid(ctx, m) + + // Sleep for time > sessionIdleTimeout, so the session expires + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m) + + // Set the keepalive handler + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m, simulator.DefaultLogin)) + + // Start the handler + Expect(m.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + time.Sleep(sessionCheckPause) + + // Session should not have been expired + assertSoapSessionValid(ctx, m) + + Expect(m.Logout(ctx)).To(Succeed()) + assertSoapSessionExpired(ctx, m) + }) + }) + }) + + getNewSessionManager := func(url *url.URL) *session.Manager { + c2, err := vim25.NewClient(ctx, soap.NewClient(url, true)) + Expect(err).NotTo(HaveOccurred()) + Expect(c2).NotTo(BeNil()) + + // With default keepalive handler + m2 := session.NewManager(c2) + + c2.RoundTripper = keepalive.NewHandlerSOAP(c2.RoundTripper, keepAliveIdle, nil) + + // Start the handler + Expect(m2.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + return m2 + } + + // To test NotAuthenticated Fault: + // We cannot "self-terminate" a session. So, this following two tests creates _two_ session managers. One with + // the custom keepalive handler, and another session to orchestrate the "NotAuthenticated" fault. + // We terminate the first session using the second session's manager. + + Context("Re-login on NotAuthenticated: with a session that is terminated", func() { + Context("when handler is called with correct userInfo", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + + // With custom keepalive handler + m1 := session.NewManager(c) + // Orchestrator session + m2 := getNewSessionManager(c.URL()) + + // Session should be valid + assertSoapSessionValid(ctx, m1) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m1) + + // set the keepalive handler + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m1, simulator.DefaultLogin)) + + // Start the handler + Expect(m1.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Wait for the keepalive handler to get called + time.Sleep(sessionCheckPause) + + // Session should be valid since Login starts a new one + assertSoapSessionValid(ctx, m1) + + // Terminate session to emulate NotAuthenticated Error + sess, err := m1.UserSession(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(sess).NotTo(BeNil()) + + By("terminating the session") + Expect(m2.TerminateSession(ctx, []string{sess.Key})).To(Succeed()) + + // session expired since it is terminated + assertSoapSessionExpired(ctx, m1) + + time.Sleep(sessionCheckPause) + + // keepalive handler must have re-logged in + assertSoapSessionValid(ctx, m1) + }) + }) + }) + }) + + Context("Relogin on NotAuthenticated: with a session that is terminated", func() { + Context("when handler is called with wrong userInfo", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + + // With custom keepalive handler + m1 := session.NewManager(c) + // With default keepalive handler + m2 := getNewSessionManager(c.URL()) + + // Session should be valid + assertSoapSessionValid(ctx, m1) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m1) + + // set the keepalive handler with wrong userInfo + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m1, nil)) + + // Start the handler + Expect(m1.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Wait for the keepalive handler to get called + time.Sleep(sessionCheckPause) + + // Session should be valid since Login starts a new one + assertSoapSessionValid(ctx, m1) + + // Terminate session to emulate NotAuthenticated Error + sess, err := m1.UserSession(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(sess).NotTo(BeNil()) + + By("terminating the session") + Expect(m2.TerminateSession(ctx, []string{sess.Key})).To(Succeed()) + + // session expired since it is terminated + assertSoapSessionExpired(ctx, m1) + + // Wait for keepalive handler to be called + time.Sleep(sessionCheckPause) + + // keepalive handler should error out, session still invalid + assertSoapSessionExpired(ctx, m1) + }) + }) + }) + }) + }) + + assertRestSessionValid := func(ctx context.Context, c *rest.Client) { + s, err := c.Session(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(s).NotTo(BeNil()) + } + + assertRestSessionExpired := func(ctx context.Context, c *rest.Client) { + s, err := c.Session(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(s).To(BeNil()) + } + + Context("REST sessions", func() { + Context("When a REST session is logged out)", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, vc *vim25.Client) { + c := rest.NewClient(vc) + + Expect(c.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Session should be valid + assertRestSessionValid(ctx, c) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertRestSessionExpired(ctx, c) + + // Set the keepalive handler + c.Transport = keepalive.NewHandlerREST(c, keepAliveIdle, RestKeepAliveHandlerFn(c, simulator.DefaultLogin)) + + // Start the handler + Expect(c.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Session should not have been expired + assertRestSessionValid(ctx, c) + + Expect(c.Logout(ctx)).To(Succeed()) + assertRestSessionExpired(ctx, c) + }) + }) + }) + }) +}) + +var _ = Describe("NewClient", func() { + Context("When called with valid config", func() { + Specify("returns a valid client and no error", func() { + client, err := NewClient(ctx, testConfig(server.URL.Hostname(), server.URL.Port(), "some-username", "some-password")) + Expect(err).ToNot(HaveOccurred()) + Expect(client).ToNot(BeNil()) + }) + }) + + Context("When called with invalid host and port", func() { + Specify("soap.ParseURL should fail", func() { + failConfig := testConfig("test%test", "", "test-user", "test-pass") + client, err := NewClient(ctx, failConfig) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to parse")) + Expect(client).To(BeNil()) + }) + }) + + Context("When called with invalid VC PNID", func() { + Specify("returns failed to parse error", func() { + failConfig := testConfig("test-pnid", "test-port", "test-user", "test-pass") + client, err := NewClient(ctx, failConfig) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid")) + Expect(err.Error()).To(ContainSubstring("port")) + Expect(err.Error()).To(ContainSubstring("test-port")) + Expect(client).To(BeNil()) + }) + }) + + DescribeTable("Should fail if given wrong username and/or wrong password", + func(expectedUsername, expectedPassword, username, password string) { + server.URL.User = url.UserPassword(expectedUsername, expectedPassword) + model.Service.Listen = server.URL + config := testConfig(server.URL.Hostname(), server.URL.Port(), username, password) + client, err := NewClient(ctx, config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("login failed for url")) + Expect(client).To(BeNil()) + }, + Entry("with wrong username and password", "correct-username", "correct-password", "username", "password"), + Entry("with wrong username and correct password", "correct-username", "correct-password", "username", "correct-password"), + Entry("with correct username and wrong password", "correct-username", "correct-password", "correct-username", "password"), + ) +}) + +// Most of the other VM Operator tests run without TLS verification. Start up a separate simulator with a fresh TLS key/cert +// +// and ensure the client can connect to it. +var _ = Describe("Tests for client TLS", func() { + Context("when the client recognizes the certificate presented by the VC", func() { + It("successfully connects to the VC", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + config.InsecureSkipTLSVerify = false + config.CAFilePath = tlsServerCertPath + _, err := NewClient(context.Background(), config) + + Expect(err).NotTo(HaveOccurred()) + }) + }) + Context("when the CA bundle referred to does not exist", func() { + It("returns an error about loading the CA bundle", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + config.CAFilePath = "/a/nonexistent/ca-bundle.crt" + config.InsecureSkipTLSVerify = false + _, err := NewClient(context.Background(), config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to set root CA")) + }) + + }) + Context("when the client does not recognize the certificate presented by the VC", func() { + Context("when TLS verification is used", func() { + It("returns an error about certificate validation", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + _, randomCANotForServer := test.GenerateSelfSignedCert() + config.CAFilePath = randomCANotForServer + config.InsecureSkipTLSVerify = false + _, err := NewClient(context.Background(), config) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("x509: certificate signed by unknown authority")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go new file mode 100644 index 000000000..7e8208443 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go @@ -0,0 +1,8 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import logf "sigs.k8s.io/controller-runtime/pkg/log" + +var log = logf.Log.WithName("vsphere").WithName("clustermodules") diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go new file mode 100644 index 000000000..ce21d3dd6 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go @@ -0,0 +1,127 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import ( + "context" + + "github.com/vmware/govmomi/vapi/cluster" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type Provider interface { + CreateModule(ctx context.Context, clusterRef types.ManagedObjectReference) (string, error) + DeleteModule(ctx context.Context, moduleID string) error + DoesModuleExist(ctx context.Context, moduleID string, cluster types.ManagedObjectReference) (bool, error) + + IsMoRefModuleMember(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) (bool, error) + AddMoRefToModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error + RemoveMoRefFromModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error +} + +type provider struct { + manager *cluster.Manager +} + +func NewProvider(restClient *rest.Client) Provider { + return &provider{ + manager: cluster.NewManager(restClient), + } +} + +func (cm *provider) CreateModule(ctx context.Context, clusterRef types.ManagedObjectReference) (string, error) { + log.Info("Creating cluster module", "cluster", clusterRef) + + moduleID, err := cm.manager.CreateModule(ctx, clusterRef) + if err != nil { + return "", err + } + + log.Info("Created cluster module", "moduleID", moduleID) + return moduleID, nil +} + +func (cm *provider) DeleteModule(ctx context.Context, moduleID string) error { + log.Info("Deleting cluster module", "moduleID", moduleID) + + err := cm.manager.DeleteModule(ctx, moduleID) + if err != nil && !lib.IsNotFoundError(err) { + return err + } + + log.Info("Deleted cluster module", "moduleID", moduleID) + return nil +} + +func (cm *provider) DoesModuleExist(ctx context.Context, moduleID string, clusterRef types.ManagedObjectReference) (bool, error) { + log.V(4).Info("Checking if cluster module exists", "moduleID", moduleID, "clusterRef", clusterRef) + + if moduleID == "" { + return false, nil + } + + // This is not efficient for as we use DoesModuleExist(). + modules, err := cm.manager.ListModules(ctx) + if err != nil { + return false, err + } + + for _, mod := range modules { + if mod.Cluster == clusterRef.Value && mod.Module == moduleID { + return true, nil + } + } + + log.V(4).Info("Cluster module doesn't exist", "moduleID", moduleID, "clusterRef", clusterRef) + return false, nil +} + +func (cm *provider) IsMoRefModuleMember(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) (bool, error) { + moduleMembers, err := cm.manager.ListModuleMembers(ctx, moduleID) + if err != nil { + return false, err + } + + for _, member := range moduleMembers { + if member.Reference() == moRef.Reference() { + return true, nil + } + } + + return false, nil +} + +func (cm *provider) AddMoRefToModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error { + isMember, err := cm.IsMoRefModuleMember(ctx, moduleID, moRef) + if err != nil { + return err + } + + if !isMember { + log.Info("Adding moRef to cluster module", "moduleID", moduleID, "moRef", moRef) + // TODO: Should we just skip the IsMoRefModuleMember() and always call this since we're already + // ignoring the first return value? + _, err := cm.manager.AddModuleMembers(ctx, moduleID, moRef.Reference()) + if err != nil { + return err + } + } + + return nil +} + +func (cm *provider) RemoveMoRefFromModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error { + log.Info("Removing moRef from cluster module", "moduleID", moduleID, "moRef", moRef) + + _, err := cm.manager.RemoveModuleMembers(ctx, moduleID, moRef) + if err != nil { + return err + } + + log.Info("Removed moRef from cluster module", "moduleID", moduleID, "moRef", moRef) + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go new file mode 100644 index 000000000..e46a72346 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ClusterModules Provider", cmTests) +} + +var suite = builder.NewTestSuite() + +func TestClusterModules(t *testing.T) { + suite.Register(t, "vSphere Provider Cluster Modules Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go new file mode 100644 index 000000000..67ce19011 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func cmTests() { + Describe("Cluster Modules", func() { + + var ( + ctx *builder.TestContextForVCSim + cmProvider clustermodules.Provider + + moduleGroup string + moduleSpec *vmopv1.ClusterModuleSpec + moduleStatus *vmopv1.ClusterModuleStatus + clusterRef types.ManagedObjectReference + vmRef types.ManagedObjectReference + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + cmProvider = clustermodules.NewProvider(ctx.RestClient) + + clusterRef = ctx.GetSingleClusterCompute().Reference() + + moduleGroup = "controller-group" + moduleSpec = &vmopv1.ClusterModuleSpec{ + GroupName: moduleGroup, + } + + moduleID, err := cmProvider.CreateModule(ctx, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(moduleID).ToNot(BeEmpty()) + + moduleStatus = &vmopv1.ClusterModuleStatus{ + GroupName: moduleSpec.GroupName, + ModuleUuid: moduleID, + } + + // TODO: Create VM instead of using one that vcsim creates for free. + vm, err := ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + vmRef = vm.Reference() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Create a ClusterModule, verify it exists and delete it", func() { + exists, err := cmProvider.DoesModuleExist(ctx, moduleStatus.ModuleUuid, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + + Expect(cmProvider.DeleteModule(ctx, moduleStatus.ModuleUuid)).To(Succeed()) + + exists, err = cmProvider.DoesModuleExist(ctx, moduleStatus.ModuleUuid, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + Context("ClusterModule-VM association", func() { + It("check membership doesn't exist", func() { + isMember, err := cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeFalse()) + }) + + It("Associate a VM with a clusterModule, check the membership and remove it", func() { + By("Associate VM") + err := cmProvider.AddMoRefToModule(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + + By("Verify membership") + isMember, err := cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeTrue()) + + By("Remove the association") + err = cmProvider.RemoveMoRefFromModule(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + + By("Verify no longer a member") + isMember, err = cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeFalse()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go new file mode 100644 index 000000000..898a7d82e --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go @@ -0,0 +1,62 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import ( + "context" + + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +// FindClusterModuleUUID returns the index in the Status.ClusterModules and UUID of the +// VC cluster module for the given groupName and cluster reference. +func FindClusterModuleUUID( + groupName string, + clusterRef types.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (int, string) { + + // Prior to the stretched cluster work, the status did not contain the VC cluster the module was + // created for, but we still need to return existing modules when the FSS is not enabled. + matchCluster := lib.IsWcpFaultDomainsFSSEnabled() + + for i, modStatus := range resourcePolicy.Status.ClusterModules { + if modStatus.GroupName == groupName && (modStatus.ClusterMoID == clusterRef.Value || !matchCluster) { + return i, modStatus.ModuleUuid + } + } + + return -1, "" +} + +// ClaimClusterModuleUUID tries to find an existing entry in the Status.ClusterModules that is for +// the given groupName and cluster reference. This is meant for after an upgrade where the FaultDomains +// FSS is now enabled but we had not previously set the ClusterMoID. +func ClaimClusterModuleUUID( + ctx context.Context, + clusterModProvider Provider, + groupName string, + clusterRef types.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (int, string, error) { + + var errs []error + + if lib.IsWcpFaultDomainsFSSEnabled() { + for i, modStatus := range resourcePolicy.Status.ClusterModules { + if modStatus.GroupName == groupName && modStatus.ClusterMoID == "" { + exists, err := clusterModProvider.DoesModuleExist(ctx, modStatus.ModuleUuid, clusterRef) + if err != nil { + errs = append(errs, err) + } else if exists { + return i, modStatus.ModuleUuid, nil + } + } + } + } + + return -1, "", k8serrors.NewAggregate(errs) +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go new file mode 100644 index 000000000..db3e83241 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" +) + +var _ = Describe("FindClusterModuleUUID", func() { + const ( + groupName1, groupName2 = "groupName1", "groupName2" + moduleUUID1, moduleUUID2 = "uuid1", "uuid2" + ) + + var ( + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + clusterRef1, clusterRef2 types.ManagedObjectReference + ) + + BeforeEach(func() { + resourcePolicy = &vmopv1.VirtualMachineSetResourcePolicy{ + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ClusterModuleGroups: []string{groupName1, groupName2}, + }, + } + + clusterRef1 = types.ManagedObjectReference{Value: "dummy-cluster1"} + clusterRef2 = types.ManagedObjectReference{Value: "dummy-cluster2"} + }) + + Context("FaultDomains FSS is disabled", func() { + + Context("GroupName does not exist", func() { + It("Returns expected values", func() { + idx, uuid := clustermodules.FindClusterModuleUUID("does-not-exist", clusterRef1, resourcePolicy) + Expect(idx).To(Equal(-1)) + Expect(uuid).To(BeEmpty()) + }) + }) + + Context("GroupName exists", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName2, + ModuleUuid: moduleUUID2, + ClusterMoID: "this should be ignored", + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName2, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + + }) + + Context("FaultDomains FSS is enabled", func() { + var ( + oldFaultDomainsFunc func() bool + ) + + BeforeEach(func() { + oldFaultDomainsFunc = lib.IsWcpFaultDomainsFSSEnabled + lib.IsWcpFaultDomainsFSSEnabled = func() bool { return true } + }) + + AfterEach(func() { + lib.IsWcpFaultDomainsFSSEnabled = oldFaultDomainsFunc + }) + + Context("GroupName does not exist", func() { + It("Returns expected values", func() { + idx, uuid := clustermodules.FindClusterModuleUUID("does-not-exist", clusterRef1, resourcePolicy) + Expect(idx).To(Equal(-1)) + Expect(uuid).To(BeEmpty()) + }) + }) + + Context("GroupName exists", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + ClusterMoID: clusterRef1.Value, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName2, + ModuleUuid: moduleUUID2, + ClusterMoID: clusterRef1.Value, + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName2, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + + Context("Matches by cluster reference", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + ClusterMoID: clusterRef1.Value, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID2, + ClusterMoID: clusterRef2.Value, + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName1, clusterRef2, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/config/config.go b/pkg/vmprovider/providers/vsphere2/config/config.go new file mode 100644 index 000000000..8d2071a42 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config.go @@ -0,0 +1,290 @@ +// Copyright (c) 2018-2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "context" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/pkg/errors" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" +) + +var log = logf.Log.WithName("vsphere").WithName("config") + +// VSphereVMProviderConfig represents the configuration for a Vsphere VM Provider instance. +// Contains information enabling integration with a backend vSphere instance for VM management. +type VSphereVMProviderConfig struct { + VcPNID string + VcPort string + VcCreds *credentials.VSphereVMProviderCredentials + Datacenter string + StorageClassRequired bool // Always true in WCP env. + UseInventoryAsContentSource bool // Always false in WCP env. + CAFilePath string + InsecureSkipTLSVerify bool // Always false in WCP env. + + // These are Zone and/or Namespace specific. + ResourcePool string + Folder string + + // Only set in simulated testing env. + Datastore string + Network string +} + +const ( + DefaultVCPort = "443" + + ProviderConfigMapName = "vsphere.provider.config.vmoperator.vmware.com" + // Keys in provider ConfigMap. + vcPNIDKey = "VcPNID" + vcPortKey = "VcPort" + vcCredsSecretNameKey = "VcCredsSecretName" //nolint:gosec + datacenterKey = "Datacenter" + resourcePoolKey = "ResourcePool" + folderKey = "Folder" + datastoreKey = "Datastore" + networkNameKey = "Network" + scRequiredKey = "StorageClassRequired" + useInventoryKey = "UseInventoryAsContentSource" + insecureSkipTLSVerifyKey = "InsecureSkipTLSVerify" + caFilePathKey = "CAFilePath" + ContentSourceKey = "ContentSource" + + NetworkConfigMapName = "vmoperator-network-config" + NameserversKey = "nameservers" // Key in the NetworkConfigMapName. + SearchSuffixesKey = "searchsuffixes" // Key in the NetworkConfigMapName. +) + +// ConfigMapToProviderConfig converts the VM provider ConfigMap to a VSphereVMProviderConfig. +func ConfigMapToProviderConfig( //nolint: revive // Ignore linter error about stuttering. + configMap *corev1.ConfigMap, + vcCreds *credentials.VSphereVMProviderCredentials) (*VSphereVMProviderConfig, error) { + + vcPNID, ok := configMap.Data[vcPNIDKey] + if !ok { + return nil, errors.New("missing configMap data field VcPNID") + } + + vcPort, ok := configMap.Data[vcPortKey] + if !ok { + vcPort = DefaultVCPort + } + + scRequired := false + if s, ok := configMap.Data[scRequiredKey]; ok { + var err error + scRequired, err = strconv.ParseBool(s) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of StorageClassRequired") + } + } + + useInventory := false + if u, ok := configMap.Data[useInventoryKey]; ok { + var err error + useInventory, err = strconv.ParseBool(u) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of UseInventory") + } + } + + // Default to validating TLS. + insecureSkipTLSVerify := false + if v, ok := configMap.Data[insecureSkipTLSVerifyKey]; ok { + var err error + insecureSkipTLSVerify, err = strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of InsecureSkipTLSVerify") + } + } + + var caFilePath string + if ca, ok := configMap.Data[caFilePathKey]; !insecureSkipTLSVerify && ok { + // The value will be /etc/vmware/wcp/tls/vmca.pem. While this is from our provider ConfigMap + // it must match the volume path in our Deployment. + caFilePath = ca + } + + ret := &VSphereVMProviderConfig{ + VcPNID: vcPNID, + VcPort: vcPort, + VcCreds: vcCreds, + Datacenter: configMap.Data[datacenterKey], + ResourcePool: configMap.Data[resourcePoolKey], + Folder: configMap.Data[folderKey], + Datastore: configMap.Data[datastoreKey], + Network: configMap.Data[networkNameKey], + StorageClassRequired: scRequired, + UseInventoryAsContentSource: useInventory, + InsecureSkipTLSVerify: insecureSkipTLSVerify, + CAFilePath: caFilePath, + } + + return ret, nil +} + +func configMapToProviderCredentials( + client ctrlruntime.Client, + configMap *corev1.ConfigMap) (*credentials.VSphereVMProviderCredentials, error) { + + secretName := configMap.Data[vcCredsSecretNameKey] + if secretName == "" { + return nil, errors.Errorf("%s creds secret not set in vmop system namespace", vcCredsSecretNameKey) + } + + return credentials.GetProviderCredentials(client, configMap.Namespace, secretName) +} + +func GetDNSInformationFromConfigMap(client ctrlruntime.Client) ([]string, []string, error) { + vmopNamespace, err := lib.GetVMOpNamespaceFromEnv() + if err != nil { + return nil, nil, err + } + + configMap := &corev1.ConfigMap{} + configMapKey := ctrlruntime.ObjectKey{Name: NetworkConfigMapName, Namespace: vmopNamespace} + if err := client.Get(context.Background(), configMapKey, configMap); err != nil { + return nil, nil, err + } + + var ( + nameservers []string + searchSuffixes []string + ) + + nsStr, ok := configMap.Data[NameserversKey] + if !ok { + return nil, nil, errors.Wrapf(err, "invalid %v ConfigMap, missing key nameservers", NetworkConfigMapName) + } + + nameservers = strings.Fields(nsStr) + if len(nameservers) == 0 { + return nil, nil, errors.Errorf("No nameservers in %v ConfigMap", NetworkConfigMapName) + } + + if len(nameservers) == 1 && nameservers[0] == "" { + return nil, nil, errors.Errorf("No valid nameservers in %v ConfigMap. It still contains key", NetworkConfigMapName) + } + + if ssStr, ok := configMap.Data[SearchSuffixesKey]; ok { + searchSuffixes = strings.Fields(ssStr) + } + + // do we need to validate that these look like valid ipv4 addresses? + return nameservers, searchSuffixes, nil +} + +// getProviderConfigMap returns the provider ConfigMap. +func getProviderConfigMap( + ctx context.Context, + client ctrlruntime.Client) (*corev1.ConfigMap, error) { + + vmopNamespace, err := lib.GetVMOpNamespaceFromEnv() + if err != nil { + return nil, err + } + + configMap := &corev1.ConfigMap{} + configMapKey := ctrlruntime.ObjectKey{Name: ProviderConfigMapName, Namespace: vmopNamespace} + if err := client.Get(ctx, configMapKey, configMap); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "error retrieving the provider ConfigMap %s", configMapKey) + } + + return configMap, nil +} + +// GetProviderConfig returns a provider config constructed from vSphere Provider ConfigMap in the VM Operator namespace. +func GetProviderConfig( + ctx context.Context, + client ctrlruntime.Client) (*VSphereVMProviderConfig, error) { + + configMap, err := getProviderConfigMap(ctx, client) + if err != nil { + return nil, err + } + + vcCreds, err := configMapToProviderCredentials(client, configMap) + if err != nil { + return nil, err + } + + providerConfig, err := ConfigMapToProviderConfig(configMap, vcCreds) + if err != nil { + return nil, err + } + + return providerConfig, nil +} + +func setConfigMapData(configMap *corev1.ConfigMap, config *VSphereVMProviderConfig, vcCredsSecretName string) { + if configMap.Data == nil { + configMap.Data = map[string]string{} + } + + configMap.Data[vcPNIDKey] = config.VcPNID + configMap.Data[vcPortKey] = config.VcPort + configMap.Data[vcCredsSecretNameKey] = vcCredsSecretName + configMap.Data[datacenterKey] = config.Datacenter + configMap.Data[resourcePoolKey] = config.ResourcePool + configMap.Data[folderKey] = config.Folder + configMap.Data[datastoreKey] = config.Datastore + configMap.Data[scRequiredKey] = strconv.FormatBool(config.StorageClassRequired) + configMap.Data[useInventoryKey] = strconv.FormatBool(config.UseInventoryAsContentSource) + configMap.Data[caFilePathKey] = config.CAFilePath + configMap.Data[insecureSkipTLSVerifyKey] = strconv.FormatBool(config.InsecureSkipTLSVerify) +} + +// ProviderConfigToConfigMap returns the ConfigMap for the config. +// Used only in testing. +func ProviderConfigToConfigMap( + namespace string, + config *VSphereVMProviderConfig, + vcCredsSecretName string) *corev1.ConfigMap { + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProviderConfigMapName, + Namespace: namespace, + }, + } + setConfigMapData(configMap, config, vcCredsSecretName) + + return configMap +} + +// UpdateVcInConfigMap updates the ConfigMap with the new vCenter PNID and Port. Returns false if no updated needed. +func UpdateVcInConfigMap(ctx context.Context, client ctrlruntime.Client, vcPNID, vcPort string) (bool, error) { + configMap, err := getProviderConfigMap(ctx, client) + if err != nil { + return false, err + } + + if configMap.Data[vcPNIDKey] == vcPNID && configMap.Data[vcPortKey] == vcPort { + // No update needed. + return false, nil + } + + origConfigMap := configMap.DeepCopy() + configMap.Data[vcPNIDKey] = vcPNID + configMap.Data[vcPortKey] = vcPort + + err = client.Patch(ctx, configMap, ctrlruntime.MergeFrom(origConfigMap)) + if err != nil { + log.Error(err, "Failed to update provider ConfigMap", "configMapName", configMap.Name) + return false, err + } + + return true, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go b/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go new file mode 100644 index 000000000..c82c0e933 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("Config", configTests) +} + +var suite = builder.NewTestSuite() + +func TestConfig(t *testing.T) { + suite.Register(t, "vSphere Provider Config Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/config/config_test.go b/pkg/vmprovider/providers/vsphere2/config/config_test.go new file mode 100644 index 000000000..28bda78bb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func configTests() { + + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("GetProviderConfig", func() { + + Context("GetProviderConfig", func() { + + Context("when a secret doesn't exist", func() { + It("returns no provider config and an error", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmop-vcsim-dummy-creds", + Namespace: ctx.PodNamespace, + }, + } + Expect(ctx.Client.Delete(ctx, secret)).To(Succeed()) + + providerConfig, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).To(HaveOccurred()) + Expect(providerConfig).To(BeNil()) + }) + }) + + Context("when a secret exists", func() { + It("returns a good provider config", func() { + _, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) + }) + + Describe("UpdateVcInConfigMap", func() { + + Context("UpdateVcInConfigMap", func() { + DescribeTable("Update VC PNID and VC Port", + func(newPnid string, newPort string) { + expectedUpdated := newPnid != "" || newPort != "" + + providerConfig, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).ToNot(HaveOccurred()) + if newPnid == "" { + newPnid = providerConfig.VcPNID + } + if newPort == "" { + newPort = providerConfig.VcPort + } + + updated, err := config.UpdateVcInConfigMap(ctx, ctx.Client, newPnid, newPort) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(Equal(expectedUpdated)) + + providerConfig, err = config.GetProviderConfig(ctx, ctx.Client) + Expect(err).NotTo(HaveOccurred()) + Expect(providerConfig.VcPNID).To(Equal(newPnid)) + Expect(providerConfig.VcPort).To(Equal(newPort)) + }, + Entry("only VC PNID is updated", "some-pnid", nil), + Entry("only VC Port is updated", nil, "some-port"), + Entry("both VC PNID and Port are updated", "some-pnid", "some-port"), + Entry("neither VC PNID and Port are updated", nil, nil), + ) + }) + }) +} + +var _ = Describe("ConfigMapToProviderConfig", func() { + + var ( + providerCreds *credentials.VSphereVMProviderCredentials + providerConfigIn *config.VSphereVMProviderConfig + configMap *corev1.ConfigMap + ) + + BeforeEach(func() { + providerCreds = &credentials.VSphereVMProviderCredentials{Username: "username", Password: "password"} + providerConfigIn = &config.VSphereVMProviderConfig{ + VcPNID: "my-vc.vmware.com", + VcPort: "433", + VcCreds: providerCreds, + Datacenter: "datacenter-42", + StorageClassRequired: false, + UseInventoryAsContentSource: false, + CAFilePath: "/etc/pki/tls/certs/ca-bundle.crt", + InsecureSkipTLSVerify: false, + ResourcePool: "resourcepool-42", + Folder: "folder-42", + Datastore: "/DC0/datastore/LocalDS_0", + } + }) + + JustBeforeEach(func() { + configMap = config.ProviderConfigToConfigMap("dummy-ns", providerConfigIn, "dummy-secrets") + }) + + It("provider config is correctly extracted from the ConfigMap", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.VcPNID).To(Equal(configMap.Data["VcPNID"])) + Expect(providerConfig.VcPort).To(Equal(configMap.Data["VcPort"])) + }) + + Context("when VcPNID is unset in configMap", func() { + It("return an error", func() { + delete(configMap.Data, "VcPNID") + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).To(HaveOccurred()) + Expect(providerConfig).To(BeNil()) + }) + }) + + Context("StorageClassRequired", func() { + It("StorageClassRequired is unset in configMap", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.StorageClassRequired).To(BeFalse()) + }) + + Context("StorageClassRequired is set in configMap", func() { + BeforeEach(func() { + providerConfigIn.StorageClassRequired = true + }) + + It("StorageClassRequired is true in config", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.StorageClassRequired).To(BeTrue()) + }) + }) + }) + + Describe("Tests for TLS configuration", func() { + + Context("when no TLS configuration is specified", func() { + It("defaults to using TLS with the system root CA", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).NotTo(HaveOccurred()) + + Expect(providerConfig.InsecureSkipTLSVerify).To(BeFalse()) + Expect(providerConfig.CAFilePath).To(Equal("/etc/pki/tls/certs/ca-bundle.crt")) + }) + }) + + Context("when the config chooses to ignore TLS verification", func() { + BeforeEach(func() { + providerConfigIn.InsecureSkipTLSVerify = true + }) + + It("sets the insecure flag in the provider config", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).NotTo(HaveOccurred()) + Expect(providerConfig.InsecureSkipTLSVerify).To(BeTrue()) + }) + }) + + Context("when the TLS settings in the Config do not pars", func() { + It("returns an error when parsing the ConfigMa", func() { + configMap.Data["InsecureSkipTLSVerify"] = "bogus" + _, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when the config chooses to use TLS verification and overrides the CA file path", func() { + BeforeEach(func() { + providerConfigIn.CAFilePath = "/etc/a/new/ca/bundle.crt" + }) + + It("uses the new CA path", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.CAFilePath).To(Equal("/etc/a/new/ca/bundle.crt")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/constants/constants.go b/pkg/vmprovider/providers/vsphere2/constants/constants.go new file mode 100644 index 000000000..ef26052b7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/constants/constants.go @@ -0,0 +1,137 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package constants + +import ( + "github.com/vmware-tanzu/vm-operator/pkg" +) + +const ( + ExtraConfigTrue = "TRUE" + ExtraConfigFalse = "FALSE" + ExtraConfigUnset = "" + ExtraConfigGuestInfoPrefix = "guestinfo." + + // VCVMAnnotation Annotation placed on the VM. + VCVMAnnotation = "Virtual Machine managed by the vSphere Virtual Machine service" + + // ManagedByExtensionKey and ManagedByExtensionType represent the ManagedBy field on the VM. + // Historically, this field was used to differentiate VM Service managed VMs from traditional ones. + ManagedByExtensionKey = "com.vmware.vcenter.wcp" + ManagedByExtensionType = "VirtualMachine" + + // VSphereCustomizationBypassKey Annotation to skip applying VMware Tools Guest Customization. + VSphereCustomizationBypassKey = pkg.VMOperatorKey + "/vsphere-customization" + VSphereCustomizationBypassDisable = "disable" + + // VMOperatorV1Alpha1ExtraConfigKey Special ExtraConfig key for v1alpha1 images. + VMOperatorV1Alpha1ExtraConfigKey = "guestinfo.vmservice.defer-cloud-init" + VMOperatorV1Alpha1ConfigReady = "ready" + VMOperatorV1Alpha1ConfigEnabled = "enabled" + + // GOSCPendingExtraConfigKey and GOSCIgnoreToolsCheckExtraConfigKey are GOSC Related ExtraConfig keys. + GOSCPendingExtraConfigKey = "tools.deployPkg.fileName" + GOSCIgnoreToolsCheckExtraConfigKey = "vmware.tools.gosc.ignoretoolscheck" + + // EnableDiskUUIDExtraConfigKey Enable UUID ExtraConfig key. + EnableDiskUUIDExtraConfigKey = "disk.enableUUID" + + // MMPowerOffVMExtraConfigKey ExtraConfig key to enable DRS to powerOff VMs when underlying host enters into + // maintenance mode. This is to ensure the maintenance mode workflow is consistent for VMs with vGPU/DDPIO devices. + MMPowerOffVMExtraConfigKey = "maintenance.vm.evacuation.poweroff" + + // NetPlanVersion points to the version used for Network config. + // For more information, please see https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v2.html + NetPlanVersion = 2 + + // VMImageCLVersionAnnotation VirtualMachineImage annotation to cache the last fetched version. + VMImageCLVersionAnnotation = pkg.VMOperatorKey + "/content-library-version" + // VMImageCLVersionAnnotationVersion is the version of the VMImageCLVersionAnnotation for the VirtualMachineImage. + VMImageCLVersionAnnotationVersion = 1 + + PCIPassthruMMIOOverrideAnnotation = pkg.VMOperatorKey + "/pci-passthru-64bit-mmio-size" + PCIPassthruMMIOExtraConfigKey = "pciPassthru.use64bitMMIO" //nolint:gosec + PCIPassthruMMIOSizeExtraConfigKey = "pciPassthru.64bitMMIOSizeGB" //nolint:gosec + PCIPassthruMMIOSizeDefault = "512" + + // MinSupportedHWVersionForPVC is the supported virtual hardware version for persistent volumes. + MinSupportedHWVersionForPVC = 15 + // MinSupportedHWVersionForPCIPassthruDevices is the supported virtual hardware version for NVidia PCI devices. + MinSupportedHWVersionForPCIPassthruDevices = 17 + + // FirmwareOverrideAnnotation is the annotation key used for firmware override. + FirmwareOverrideAnnotation = pkg.VMOperatorKey + "/firmware" + + CloudInitTypeAnnotation = pkg.VMOperatorKey + "/cloudinit-type" + CloudInitTypeValueCloudInitPrep = "cloudinitprep" + CloudInitTypeValueGuestInfo = "guestinfo" + + CloudInitGuestInfoMetadata = "guestinfo.metadata" + CloudInitGuestInfoMetadataEncoding = "guestinfo.metadata.encoding" + CloudInitGuestInfoUserdata = "guestinfo.userdata" + CloudInitGuestInfoUserdataEncoding = "guestinfo.userdata.encoding" + + // InstanceStoragePVCNamePrefix prefix of auto-generated PVC names. + InstanceStoragePVCNamePrefix = "instance-pvc-" + // InstanceStorageLabelKey identifies resources related to instance storage. + // The primary purpose of this label is to identify instance storage resources such as + // PVCs and CNSNodeVMAttachments but not for List and kubectl-get of VM resources. + InstanceStorageLabelKey = "vmoperator.vmware.com/instance-storage-resource" + // InstanceStoragePVCsBoundAnnotationKey annotation key used to set bound state of all instance storage PVCs. + InstanceStoragePVCsBoundAnnotationKey = "vmoperator.vmware.com/instance-storage-pvcs-bound" + // InstanceStoragePVPlacementErrorAnnotationKey annotation key to set PV creation error. + // CSI reference to this annotation where it is defined: + // https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/pkg/syncer/k8scloudoperator/placement.go + InstanceStoragePVPlacementErrorAnnotationKey = "failure-domain.beta.vmware.com/storagepool" + // InstanceStorageSelectedNodeMOIDAnnotationKey value corresponds to MOID of ESXi node that is elected to place instance storage volumes. + InstanceStorageSelectedNodeMOIDAnnotationKey = "vmoperator.vmware.com/instance-storage-selected-node-moid" + // InstanceStorageSelectedNodeAnnotationKey value corresponds to FQDN of ESXi node that is elected to place instance storage volumes. + InstanceStorageSelectedNodeAnnotationKey = "vmoperator.vmware.com/instance-storage-selected-node" + // KubernetesSelectedNodeAnnotationKey annotation key to set selected node on PVC. + KubernetesSelectedNodeAnnotationKey = "volume.kubernetes.io/selected-node" + // InstanceStoragePVPlacementErrorPrefix indicates prefix of error value. + InstanceStoragePVPlacementErrorPrefix = "FAILED_" + // InstanceStorageNotEnoughResErr is an error constant to indicate not enough resources. + InstanceStorageNotEnoughResErr = "FAILED_PLACEMENT-NotEnoughResources" + // InstanceStorageVDiskID vDisk ID for instance storage volume. + InstanceStorageVDiskID = "cc737f33-2aa3-4594-aa60-df7d6d4cb984" + + // XsiNamespace indicates the XML scheme instance namespace. + XsiNamespace = "http://www.w3.org/2001/XMLSchema-instance" + // ConfigSpecProviderXML indicates XML as the config spec transport type for virtual machine deployment. + ConfigSpecProviderXML = "XML" + + // V1alpha1FirstIP is an alias for versioned templating function V1alpha1_FirstIP. + V1alpha1FirstIP = "V1alpha1_FirstIP" + // V1alpha1FirstNicMacAddr is an alias for versioned templating function V1alpha1_FirstNicMacAddr. + V1alpha1FirstNicMacAddr = "V1alpha1_FirstNicMacAddr" + // V1alpha1FirstIPFromNIC is an alias for versioned templating function V1alpha1_FirstIPFromNIC. + V1alpha1FirstIPFromNIC = "V1alpha1_FirstIPFromNIC" + // V1alpha1IPsFromNIC is an alias for versioned templating function V1alpha1_IPsFromNIC. + V1alpha1IPsFromNIC = "V1alpha1_IPsFromNIC" + // V1alpha1FormatIP is an alias for versioned templating function V1alpha1_FormatIP. + V1alpha1FormatIP = "V1alpha1_FormatIP" + // V1alpha1IP is an alias for versioned templating function V1alpha1_IP. + V1alpha1IP = "V1alpha1_IP" + // V1alpha1SubnetMask is an alias for versioned templating function V1alpha1_SubnetMask. + V1alpha1SubnetMask = "V1alpha1_SubnetMask" + // V1alpha1FormatNameservers is an alias for versioned templating function V1alpha1_FormatNameservers. + V1alpha1FormatNameservers = "V1alpha1_FormatNameservers" + // V1alpha2FirstIP is an alias for versioned templating function V1alpha2_FirstIP. + V1alpha2FirstIP = "V1alpha2_FirstIP" + // V1alpha2FirstNicMacAddr is an alias for versioned templating function V1alpha2_FirstNicMacAddr. + V1alpha2FirstNicMacAddr = "V1alpha2_FirstNicMacAddr" + // V1alpha2FirstIPFromNIC is an alias for versioned templating function V1alpha2_FirstIPFromNIC. + V1alpha2FirstIPFromNIC = "V1alpha2_FirstIPFromNIC" + // V1alpha2IPsFromNIC is an alias for versioned templating function V1alpha2_IPsFromNIC. + V1alpha2IPsFromNIC = "V1alpha2_IPsFromNIC" + // V1alpha2FormatIP is an alias for versioned templating function V1alpha2_FormatIP. + V1alpha2FormatIP = "V1alpha2_FormatIP" + // V1alpha2IP is an alias for versioned templating function V1alpha2_IP. + V1alpha2IP = "V1alpha2_IP" + // V1alpha2SubnetMask is an alias for versioned templating function V1alpha2_SubnetMask. + V1alpha2SubnetMask = "V1alpha2_SubnetMask" + // V1alpha2FormatNameservers is an alias for versioned templating function V1alpha2_FormatNameservers. + V1alpha2FormatNameservers = "V1alpha2_FormatNameservers" +) diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go new file mode 100644 index 000000000..7dc3b9f54 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go @@ -0,0 +1,10 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var log = logf.Log.WithName("vsphere").WithName("contentlibrary") diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go new file mode 100644 index 000000000..be2ff212d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go @@ -0,0 +1,379 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + "strconv" + "time" + + k8serrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/soap" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type Provider interface { + GetLibraryItems(ctx context.Context, clUUID string) ([]library.Item, error) + GetLibraryItem(ctx context.Context, libraryUUID, itemName string, + notFoundReturnErr bool) (*library.Item, error) + GetLibraryItemID(ctx context.Context, itemUUID string) (*library.Item, error) + ListLibraryItems(ctx context.Context, libraryUUID string) ([]string, error) + UpdateLibraryItem(ctx context.Context, itemID, newName string, newDescription *string) error + RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item *library.Item) (*ovf.Envelope, error) + RetrieveOvfEnvelopeByLibraryItemID(ctx context.Context, itemID string) (*ovf.Envelope, error) + + // TODO: Testing only. Remove these from this file. + CreateLibraryItem(ctx context.Context, libraryItem library.Item, path string) error +} + +type provider struct { + libMgr *library.Manager + retryInterval time.Duration +} + +const ( + EnvContentLibAPIWaitSecs = "CONTENT_API_WAIT_SECS" // BMV: Investigate if setting this to 1 actually reduces the integration test time. + DefaultContentLibAPIWaitSecs = 5 +) + +func IsSupportedDeployType(t string) bool { + switch t { + case library.ItemTypeVMTX, library.ItemTypeOVF: + // Keep in sync with what cloneVMFromContentLibrary() handles. + return true + default: + return false + } +} + +func NewProvider(restClient *rest.Client) Provider { + waitSeconds, err := strconv.Atoi(os.Getenv(EnvContentLibAPIWaitSecs)) + if err != nil || waitSeconds < 1 { + waitSeconds = DefaultContentLibAPIWaitSecs + } + + return NewProviderWithWaitSec(restClient, waitSeconds) +} + +func NewProviderWithWaitSec(restClient *rest.Client, waitSeconds int) Provider { + return &provider{ + libMgr: library.NewManager(restClient), + retryInterval: time.Duration(waitSeconds) * time.Second, + } +} + +func (cs *provider) ListLibraryItems(ctx context.Context, libraryUUID string) ([]string, error) { + logger := log.WithValues("libraryUUID", libraryUUID) + itemList, err := cs.libMgr.ListLibraryItems(ctx, libraryUUID) + if err != nil { + if lib.IsNotFoundError(err) { + logger.Error(err, "cannot list items from content library that does not exist") + return nil, nil + } + return nil, err + } + return itemList, err +} + +func (cs *provider) GetLibraryItems(ctx context.Context, libraryUUID string) ([]library.Item, error) { + logger := log.WithValues("libraryUUID", libraryUUID) + itemList, err := cs.libMgr.ListLibraryItems(ctx, libraryUUID) + if err != nil { + if lib.IsNotFoundError(err) { + logger.Error(err, "cannot list items from content library that does not exist") + return nil, nil + } + return nil, err + } + + // best effort to get content library items. + resErrs := make([]error, 0) + items := make([]library.Item, 0) + for _, itemID := range itemList { + item, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + resErrs = append(resErrs, err) + logger.Error(err, "get library item failed", "itemID", itemID) + continue + } + items = append(items, *item) + } + + return items, k8serrors.NewAggregate(resErrs) +} + +func (cs *provider) GetLibraryItem(ctx context.Context, libraryUUID, itemName string, + notFoundReturnErr bool) (*library.Item, error) { + itemIDs, err := cs.libMgr.FindLibraryItems(ctx, library.FindItem{LibraryID: libraryUUID, Name: itemName}) + if err != nil { + return nil, errors.Wrapf(err, "failed to find image: %s", itemName) + } + + if len(itemIDs) == 0 { + if notFoundReturnErr { + return nil, errors.Errorf("no library item named: %s", itemName) + } + return nil, nil + } + if len(itemIDs) != 1 { + return nil, errors.Errorf("multiple library items named: %s", itemName) + } + + item, err := cs.libMgr.GetLibraryItem(ctx, itemIDs[0]) + if err != nil { + return nil, errors.Wrapf(err, "failed to get library item: %s", itemName) + } + + return item, nil +} + +func (cs *provider) GetLibraryItemID(ctx context.Context, itemUUID string) (*library.Item, error) { + item, err := cs.libMgr.GetLibraryItem(ctx, itemUUID) + if err != nil { + return nil, errors.Wrapf(err, "failed to find image: %s", itemUUID) + } + + return item, nil +} + +// RetrieveOvfEnvelopeByLibraryItemID retrieves the OVF Envelope by the given library item ID. +func (cs *provider) RetrieveOvfEnvelopeByLibraryItemID(ctx context.Context, itemID string) (*ovf.Envelope, error) { + libItem, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + return nil, err + } + + if libItem == nil || libItem.Type != library.ItemTypeOVF { + log.Error(nil, "empty or non OVF library item type, skipping", "itemID", itemID) + // No need to return the error here to avoid unnecessary reconciliation. + return nil, nil + } + + return cs.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem) +} + +func readerFromURL(ctx context.Context, c *rest.Client, url *url.URL) (io.ReadCloser, error) { + p := soap.DefaultDownload + readerStream, _, err := c.Download(ctx, url, &p) + if err != nil { + // Log message used by VMC LINT. Refer to before making changes + log.Error(err, "Error occurred when downloading file", "url", url) + return nil, err + } + + return readerStream, nil +} + +// RetrieveOvfEnvelopeFromLibraryItem downloads the supported file from content library. +// parses the downloaded ovf and returns the OVF Envelope descriptor for consumption. +func (cs *provider) RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item *library.Item) (*ovf.Envelope, error) { + // Create a download session for the file referred to by item id. + sessionID, err := cs.libMgr.CreateLibraryItemDownloadSession(ctx, library.Session{LibraryItemID: item.ID}) + if err != nil { + return nil, err + } + + logger := log.WithValues("sessionID", sessionID, "itemID", item.ID, "itemName", item.Name) + logger.V(4).Info("download session for item created") + + defer func() { + if err := cs.libMgr.DeleteLibraryItemDownloadSession(ctx, sessionID); err != nil { + logger.Error(err, "Error deleting download session") + } + }() + + // Download ovf from the library item. + fileURL, err := cs.generateDownloadURLForLibraryItem(ctx, logger, sessionID, item) + if err != nil { + return nil, err + } + + downloadedFileContent, err := readerFromURL(ctx, cs.libMgr.Client, fileURL) + if err != nil { + logger.Error(err, "error downloading file from library item") + return nil, err + } + + logger.V(4).Info("downloaded library item") + defer func() { + _ = downloadedFileContent.Close() + }() + + envelope, err := ovf.Unmarshal(downloadedFileContent) + if err != nil { + logger.Error(err, "error parsing the OVF envelope") + return nil, nil + } + + return envelope, nil +} + +// UpdateLibraryItem updates the content library item's name and description. +func (cs *provider) UpdateLibraryItem(ctx context.Context, itemID, newName string, newDescription *string) error { + log.Info("Updating Library Item", "itemID", itemID, + "newName", newName, "newDescription", newDescription) + + item, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + log.Error(err, "error getting library item") + return err + } + + if newName != "" { + item.Name = newName + } + if newDescription != nil { + item.Description = newDescription + } + + return cs.libMgr.UpdateLibraryItem(ctx, item) +} + +// Only used in testing. +func (cs *provider) CreateLibraryItem(ctx context.Context, libraryItem library.Item, path string) error { + log.Info("Creating Library Item", "item", libraryItem, "path", path) + + itemID, err := cs.libMgr.CreateLibraryItem(ctx, libraryItem) + if err != nil { + return err + } + + sessionID, err := cs.libMgr.CreateLibraryItemUpdateSession(ctx, library.Session{LibraryItemID: itemID}) + if err != nil { + return err + } + + // Update Library item with library file "ovf" + uploadFunc := func(c *rest.Client, path string) error { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + + fi, err := f.Stat() + if err != nil { + return err + } + + info := library.UpdateFile{ + Name: filepath.Base(path), + SourceType: "PUSH", + Size: fi.Size(), + } + + update, err := cs.libMgr.AddLibraryItemFile(ctx, sessionID, info) + if err != nil { + return err + } + + u, err := url.Parse(update.UploadEndpoint.URI) + if err != nil { + return err + } + + p := soap.DefaultUpload + p.ContentLength = info.Size + + return c.Upload(ctx, f, u, &p) + } + + if err = uploadFunc(cs.libMgr.Client, path); err != nil { + return err + } + + return cs.libMgr.CompleteLibraryItemUpdateSession(ctx, sessionID) +} + +// generateDownloadURLForLibraryItem downloads the file from content library in 3 steps: +// 1. list the available files and downloads only the ovf files based on filename suffix +// 2. prepare the download session and fetch the url to be used for download +// 3. download the file. +func (cs *provider) generateDownloadURLForLibraryItem( + ctx context.Context, + logger logr.Logger, + sessionID string, + item *library.Item) (*url.URL, error) { + + // List the files available for download in the library item. + files, err := cs.libMgr.ListLibraryItemDownloadSessionFile(ctx, sessionID) + if err != nil { + return nil, err + } + + var fileToDownload string + for _, file := range files { + logger.V(4).Info("Library Item file", "fileName", file.Name) + if ext := filepath.Ext(file.Name); ext != "" && IsSupportedDeployType(ext[1:]) { + fileToDownload = file.Name + break + } + } + if fileToDownload == "" { + return nil, errors.Errorf("No files with supported deploy type are available for download for %s", item.ID) + } + + _, err = cs.libMgr.PrepareLibraryItemDownloadSessionFile(ctx, sessionID, fileToDownload) + if err != nil { + return nil, err + } + + logger.V(4).Info("request posted to prepare file", "fileToDownload", fileToDownload) + + // Content library api to prepare a file for download guarantees eventual end state of either + // ERROR or PREPARED in order to avoid posting too many requests to the api. + var fileURL string + err = wait.PollUntilContextCancel(ctx, cs.retryInterval, true, func(_ context.Context) (bool, error) { + downloadSessResp, err := cs.libMgr.GetLibraryItemDownloadSession(ctx, sessionID) + if err != nil { + return false, err + } + + if downloadSessResp.ErrorMessage != nil { + return false, downloadSessResp.ErrorMessage + } + + info, err := cs.libMgr.GetLibraryItemDownloadSessionFile(ctx, sessionID, fileToDownload) + if err != nil { + return false, err + } + + if info.Status == "ERROR" { + // Log message used by VMC LINT. Refer to before making changes + return false, errors.Errorf("Error occurred preparing file for download %v", info.ErrorMessage) + } + + if info.Status != "PREPARED" { + return false, nil + } + + if info.DownloadEndpoint == nil { + return false, errors.Errorf("Prepared file for download does not have endpoint") + } + + fileURL = info.DownloadEndpoint.URI + log.V(4).Info("Downloaded file", "fileURL", fileURL) + return true, nil + }) + + if err != nil { + return nil, err + } + + return url.Parse(fileURL) +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go new file mode 100644 index 000000000..2055b76eb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ContentLibrary Provider", clTests) +} + +var suite = builder.NewTestSuite() + +func TestContentLibrary(t *testing.T) { + suite.Register(t, "vSphere Provider ContentLibrary Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go new file mode 100644 index 000000000..968eac039 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go @@ -0,0 +1,139 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + "os" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vapi/library" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func clTests() { + Describe("Content Library", func() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + + clProvider contentlibrary.Provider + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + testConfig.WithContentLibrary = true + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + clProvider = contentlibrary.NewProvider(ctx.RestClient) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("when items are present in library", func() { + + It("List items id in library", func() { + items, err := clProvider.GetLibraryItems(ctx, ctx.ContentLibraryID) + Expect(err).ToNot(HaveOccurred()) + Expect(items).ToNot(BeEmpty()) + }) + + It("Does not return error when library does not exist", func() { + items, err := clProvider.GetLibraryItems(ctx, "dummy-cl") + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(BeEmpty()) + }) + + It("Does not return error when item name is invalid when notFoundReturnErr is set to false", func() { + item, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, "dummy-name", true) + Expect(err).To(HaveOccurred()) + Expect(item).To(BeNil()) + + item, err = clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, "dummy-name", false) + Expect(err).NotTo(HaveOccurred()) + Expect(item).To(BeNil()) + }) + + It("Gets items and returns OVF", func() { + item, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, ctx.ContentLibraryImageName, true) + Expect(err).ToNot(HaveOccurred()) + Expect(item).ToNot(BeNil()) + + ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, item) + Expect(err).ToNot(HaveOccurred()) + Expect(ovfEnvelope).ToNot(BeNil()) + }) + }) + + Context("when items are not present in library", func() { + + Context("when invalid item id is passed", func() { + + It("returns an error creating a download session", func() { + libItem := &library.Item{ + Name: "fakeItem", + Type: "ovf", + LibraryID: "fakeID", + } + + ovf, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("404 Not Found")) + Expect(ovf).To(BeNil()) + }) + }) + }) + + Context("called with an OVF that is invalid", func() { + var ovfPath string + + AfterEach(func() { + if ovfPath != "" { + Expect(os.Remove(ovfPath)).To(Succeed()) + } + }) + + It("does not return error", func() { + ovf, err := os.CreateTemp("", "fake-*.ovf") + Expect(err).NotTo(HaveOccurred()) + ovfPath = ovf.Name() + + ovfInfo, err := ovf.Stat() + Expect(err).NotTo(HaveOccurred()) + + libItemName := strings.Split(ovfInfo.Name(), ".ovf")[0] + libItem := library.Item{ + Name: libItemName, + Type: "ovf", + LibraryID: ctx.ContentLibraryID, + } + + err = clProvider.CreateLibraryItem(ctx, libItem, ovfPath) + Expect(err).NotTo(HaveOccurred()) + + libItem2, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, libItemName, true) + Expect(err).ToNot(HaveOccurred()) + Expect(libItem2).ToNot(BeNil()) + Expect(libItem2.Name).To(Equal(libItem.Name)) + + ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem2) + Expect(err).ToNot(HaveOccurred()) + Expect(ovfEnvelope).To(BeNil()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go new file mode 100644 index 000000000..b4a2fd866 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go @@ -0,0 +1,149 @@ +// Copyright (c) 2019-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + "regexp" + "strconv" + "strings" + + "github.com/vmware/govmomi/ovf" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +var vmxRe = regexp.MustCompile(`vmx-(\d+)`) + +// ParseVirtualHardwareVersion parses the virtual hardware version +// For eg. "vmx-15" returns 15. +func ParseVirtualHardwareVersion(vmxVersion string) int32 { + // obj is the full string and the submatch (\d+) and return a []string with values + obj := vmxRe.FindStringSubmatch(vmxVersion) + if len(obj) != 2 { + return 0 + } + + version, err := strconv.ParseInt(obj[1], 10, 32) + if err != nil { + return 0 + } + + return int32(version) +} + +// UpdateVmiWithOvfEnvelope updates the given vmi object with the content of given OVF envelope. +func UpdateVmiWithOvfEnvelope(vmi client.Object, ovfEnvelope ovf.Envelope) { + var status *vmopv1.VirtualMachineImageStatus + + switch vmi := vmi.(type) { + case *vmopv1.VirtualMachineImage: + status = &vmi.Status + case *vmopv1.ClusterVirtualMachineImage: + status = &vmi.Status + default: + return + } + + if ovfEnvelope.VirtualSystem != nil { + initImageStatusFromOVFVirtualSystem(status, ovfEnvelope.VirtualSystem) + + ovfSystemProps := getVmwareSystemPropertiesFromOvf(ovfEnvelope.VirtualSystem) + if len(ovfSystemProps) > 0 { + annotations := vmi.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + vmi.SetAnnotations(annotations) + } + + for k, v := range ovfSystemProps { + annotations[k] = v + } + } + } +} + +func initImageStatusFromOVFVirtualSystem( + imageStatus *vmopv1.VirtualMachineImageStatus, + ovfVirtualSystem *ovf.VirtualSystem) { + + // Use info from the first product section in the VM image, if one exists. + if product := ovfVirtualSystem.Product; len(product) > 0 { + p := product[0] + + productInfo := &imageStatus.ProductInfo + productInfo.Vendor = p.Vendor + productInfo.Product = p.Product + productInfo.Version = p.Version + productInfo.FullVersion = p.FullVersion + } + + // Use operating system info from the first os section in the VM image, if one exists. + if os := ovfVirtualSystem.OperatingSystem; len(os) > 0 { + o := os[0] + + osInfo := &imageStatus.OSInfo + osInfo.ID = strconv.Itoa(int(o.ID)) + if o.Version != nil { + osInfo.Version = *o.Version + } + if o.OSType != nil { + osInfo.Type = *o.OSType + } + } + + // Use hardware section info from the VM image, if one exists. + if virtualHW := ovfVirtualSystem.VirtualHardware; len(virtualHW) > 0 { + imageStatus.Firmware = getFirmwareType(virtualHW[0]) + + if sys := virtualHW[0].System; sys != nil && sys.VirtualSystemType != nil { + ver := ParseVirtualHardwareVersion(*sys.VirtualSystemType) + if ver != 0 { + imageStatus.HardwareVersion = &ver + } + } + } + + for _, product := range ovfVirtualSystem.Product { + for _, prop := range product.Property { + // Only show user configurable properties + if prop.UserConfigurable != nil && *prop.UserConfigurable { + property := vmopv1.OVFProperty{ + Key: prop.Key, + Type: prop.Type, + Default: prop.Default, + } + imageStatus.OVFProperties = append(imageStatus.OVFProperties, property) + } + } + } +} + +func getVmwareSystemPropertiesFromOvf(ovfVirtualSystem *ovf.VirtualSystem) map[string]string { + properties := make(map[string]string) + + if ovfVirtualSystem != nil { + for _, product := range ovfVirtualSystem.Product { + for _, prop := range product.Property { + if strings.HasPrefix(prop.Key, "vmware-system") { + if prop.Default != nil { + properties[prop.Key] = *prop.Default + } + } + } + } + } + + return properties +} + +// getFirmwareType returns the firmware type (eg: "efi", "bios") present in the virtual hardware section of the OVF. +func getFirmwareType(hardware ovf.VirtualHardwareSection) string { + for _, cfg := range hardware.Config { + if cfg.Key == "firmware" { + return cfg.Value + } + } + return "" +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go new file mode 100644 index 000000000..051f1bb91 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +var _ = Describe("ParseVirtualHardwareVersion", func() { + It("empty hardware string", func() { + vmxHwVersionString := "" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("invalid hardware string", func() { + vmxHwVersionString := "blah" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("valid hardware version string eg. vmx-15", func() { + vmxHwVersionString := "vmx-15" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(Equal(int32(15))) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials.go new file mode 100644 index 000000000..9ac994bf5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials.go @@ -0,0 +1,62 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "context" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" +) + +// VSphereVMProviderCredentials wraps the data needed to login to vCenter. +type VSphereVMProviderCredentials struct { + Username string + Password string +} + +func GetProviderCredentials(client ctrlruntime.Client, namespace, secretName string) (*VSphereVMProviderCredentials, error) { + secret := &corev1.Secret{} + secretKey := types.NamespacedName{Namespace: namespace, Name: secretName} + if err := client.Get(context.Background(), secretKey, secret); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "cannot find secret for provider credentials: %s", secretKey) + } + + var credentials VSphereVMProviderCredentials + credentials.Username = string(secret.Data["username"]) + credentials.Password = string(secret.Data["password"]) + + if credentials.Username == "" || credentials.Password == "" { + return nil, errors.New("vCenter username and password are missing") + } + + return &credentials, nil +} + +func setSecretData(secret *corev1.Secret, credentials *VSphereVMProviderCredentials) { + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + + secret.Data["username"] = []byte(credentials.Username) + secret.Data["password"] = []byte(credentials.Password) +} + +// ProviderCredentialsToSecret returns the Secret for the credentials. +// Testing only. +func ProviderCredentialsToSecret(namespace string, credentials *VSphereVMProviderCredentials, vcCredsSecretName string) *corev1.Secret { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: vcCredsSecretName, + Namespace: namespace, + }, + } + setSecretData(secret, credentials) + + return secret +} diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go new file mode 100644 index 000000000..45cedd1ea --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go @@ -0,0 +1,44 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +var ( + model *simulator.Model + server *simulator.Server + ctx context.Context + tlsTestModel *simulator.Model + tlsServer *simulator.Server + tlsServerCertPath string + tlsServerKeyPath string +) + +var _ = BeforeSuite(func() { + ctx, model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer = test.BeforeSuite() +}) + +var _ = AfterSuite(func() { + test.AfterSuite( + ctx, + model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer) +}) + +func TestCredentials(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Credentials Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go new file mode 100644 index 000000000..8950bdffa --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + + . "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func newSecret(name string, ns string, user string, pass string) (*corev1.Secret, *VSphereVMProviderCredentials) { + creds := &VSphereVMProviderCredentials{ + Username: user, + Password: pass, + } + secret := ProviderCredentialsToSecret(ns, creds, name) + return secret, creds +} + +var _ = Describe("GetProviderCredentials", func() { + + Context("when a good secret exists", func() { + Specify("returns good credentials with no error", func() { + secretIn, credsIn := newSecret("some-name", "some-namespace", "some-user", "some-pass") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).ToNot(HaveOccurred()) + Expect(credsOut).To(Equal(credsIn)) + }) + }) + + Context("when a bad secret exists", func() { + + Context("with empty username", func() { + Specify("returns no credentials with error", func() { + secretIn, _ := newSecret("some-name", "some-namespace", "", "some-pass") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) + + Context("with empty password", func() { + Specify("returns no credentials with error", func() { + secretIn, _ := newSecret("some-name", "some-namespace", "some-user", "") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) + }) + + Context("when no secret exists", func() { + Specify("returns no credentials with error", func() { + client := builder.NewFakeClient() + credsOut, err := GetProviderCredentials(client, "none-namespace", "none-name") + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go b/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go new file mode 100644 index 000000000..698faa7d5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go @@ -0,0 +1,41 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package instancestorage + +import ( + "strings" + + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +// IsPresent checks if VM Spec has instance volumes added to its Volumes list. +func IsPresent(vm *vmopv1.VirtualMachine) bool { + for _, vol := range vm.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil && pvc.InstanceVolumeClaim != nil { + return true + } + } + return false +} + +// FilterVolumes returns instance storage volumes present in VM spec. +func FilterVolumes(vm *vmopv1.VirtualMachine) []vmopv1.VirtualMachineVolume { + var volumes []vmopv1.VirtualMachineVolume + for _, vol := range vm.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil && pvc.InstanceVolumeClaim != nil { + volumes = append(volumes, vol) + } + } + + return volumes +} + +func IsInsufficientQuota(err error) bool { + if apiErrors.IsForbidden(err) && (strings.Contains(err.Error(), "insufficient quota") || strings.Contains(err.Error(), "exceeded quota")) { + return true + } + return false +} diff --git a/pkg/vmprovider/providers/vsphere2/internal/internal.go b/pkg/vmprovider/providers/vsphere2/internal/internal.go new file mode 100644 index 000000000..8a4f68da3 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/internal/internal.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//nolint:revive,stylecheck +package internal + +import ( + "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" +) + +type InvokeFSR_TaskRequest struct { + This types.ManagedObjectReference `xml:"_this"` +} + +type InvokeFSR_TaskResponse struct { + Returnval types.ManagedObjectReference `xml:"returnval"` +} + +type InvokeFSR_TaskBody struct { + Req *InvokeFSR_TaskRequest `xml:"urn:vim25 InvokeFSR_Task,omitempty"` + Res *InvokeFSR_TaskResponse `xml:"InvokeFSR_TaskResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *InvokeFSR_TaskBody) Fault() *soap.Fault { + return b.Fault_ +} + +func InvokeFSR_Task(ctx context.Context, r soap.RoundTripper, req *InvokeFSR_TaskRequest) (*InvokeFSR_TaskResponse, error) { + var reqBody, resBody InvokeFSR_TaskBody + reqBody.Req = req + if err := r.RoundTrip(ctx, &reqBody, &resBody); err != nil { + return nil, err + } + return resBody.Res, nil +} + +func VirtualMachineFSR(ctx context.Context, vm types.ManagedObjectReference, client *vim25.Client) (*object.Task, error) { + req := InvokeFSR_TaskRequest{ + This: vm, + } + res, err := InvokeFSR_Task(ctx, client, &req) + if err != nil { + return nil, err + } + return object.NewTask(client, res.Returnval), nil +} + +type CustomizationCloudinitPrep struct { + types.CustomizationIdentitySettings + + Metadata string `xml:"metadata"` + Userdata string `xml:"userdata,omitempty"` +} diff --git a/pkg/vmprovider/providers/vsphere2/network/gosc.go b/pkg/vmprovider/providers/vsphere2/network/gosc.go new file mode 100644 index 000000000..feb9f9f3a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/gosc.go @@ -0,0 +1,79 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + "net" + + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +func GuestOSCustomization(results NetworkInterfaceResults) ([]vimtypes.CustomizationAdapterMapping, error) { + mappings := make([]vimtypes.CustomizationAdapterMapping, 0, len(results.Results)) + + for _, r := range results.Results { + adapter := vimtypes.CustomizationIPSettings{ + DnsServerList: r.Nameservers, + } + + if r.DHCP4 { + adapter.Ip = &vimtypes.CustomizationDhcpIpGenerator{} + } else { + // GOSC doesn't support multiple IPv4 address per interface so use the first one. Old code + // only ever set one gateway so do the same here too. + for _, ipConfig := range r.IPConfigs { + if !ipConfig.IsIPv4 { + continue + } + + ip, ipNet, err := net.ParseCIDR(ipConfig.IPCIDR) + if err != nil { + return nil, err + } + subnetMask := net.CIDRMask(ipNet.Mask.Size()) + + adapter.Ip = &vimtypes.CustomizationFixedIp{IpAddress: ip.String()} + adapter.SubnetMask = net.IP(subnetMask).String() + adapter.Gateway = []string{ipConfig.Gateway} + break + } + } + + if r.DHCP6 { + adapter.IpV6Spec = &vimtypes.CustomizationIPSettingsIpV6AddressSpec{ + Ip: []vimtypes.BaseCustomizationIpV6Generator{ + &vimtypes.CustomizationDhcpIpV6Generator{}, + }, + } + } else { + for _, ipConfig := range r.IPConfigs { + if ipConfig.IsIPv4 { + continue + } + + ip, ipNet, err := net.ParseCIDR(ipConfig.IPCIDR) + if err != nil { + return nil, err + } + ones, _ := ipNet.Mask.Size() + + if adapter.IpV6Spec == nil { + adapter.IpV6Spec = &vimtypes.CustomizationIPSettingsIpV6AddressSpec{} + } + adapter.IpV6Spec.Ip = append(adapter.IpV6Spec.Ip, &vimtypes.CustomizationFixedIpV6{ + IpAddress: ip.String(), + SubnetMask: int32(ones), + }) + adapter.IpV6Spec.Gateway = append(adapter.IpV6Spec.Gateway, ipConfig.Gateway) + } + } + + mappings = append(mappings, vimtypes.CustomizationAdapterMapping{ + MacAddress: r.MacAddress, + Adapter: adapter, + }) + } + + return mappings, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/network/gosc_test.go b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go new file mode 100644 index 000000000..1eea859a8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +var _ = Describe("GOSC", func() { + const ( + macAddr1 = "50-8A-80-9D-28-22" + + ipv4Gateway = "192.168.1.1" + ipv4 = "192.168.1.10" + ipv4CIDR = ipv4 + "/24" + ipv6Gateway = "fd8e:b5a0:f172:123::1" + ipv6 = "fd8e:b5a0:f172:123::f" + ipv6Subnet = 48 + + dnsServer1 = "9.9.9.9" + ) + + Context("GuestOSCustomization", func() { + + var ( + results network.NetworkInterfaceResults + adapterMappings []types.CustomizationAdapterMapping + err error + ) + + BeforeEach(func() { + results.Results = nil + }) + + JustBeforeEach(func() { + adapterMappings, err = network.GuestOSCustomization(results) + }) + + Context("IPv4/6 Static adapter", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + IPCIDR: ipv4CIDR, + IsIPv4: true, + Gateway: ipv4Gateway, + }, + { + IPCIDR: ipv6 + fmt.Sprintf("/%d", ipv6Subnet), + IsIPv4: false, + Gateway: ipv6Gateway, + }, + }, + MacAddress: macAddr1, + Name: "eth0", + DHCP4: false, + DHCP6: false, + MTU: 1500, // AFAIK not supported via GOSC + Nameservers: []string{dnsServer1}, + Routes: nil, // AFAIK not supported via GOSC + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(adapterMappings).To(HaveLen(1)) + mapping := adapterMappings[0] + + adapter := mapping.Adapter + Expect(mapping.MacAddress).To(Equal(macAddr1)) + Expect(adapter.Gateway).To(Equal([]string{ipv4Gateway})) + Expect(adapter.SubnetMask).To(Equal("255.255.255.0")) + Expect(adapter.DnsServerList).To(Equal([]string{dnsServer1})) + Expect(adapter.Ip).To(BeAssignableToTypeOf(&types.CustomizationFixedIp{})) + fixedIP := adapter.Ip.(*types.CustomizationFixedIp) + Expect(fixedIP.IpAddress).To(Equal(ipv4)) + + ipv6Spec := adapter.IpV6Spec + Expect(ipv6Spec).ToNot(BeNil()) + Expect(ipv6Spec.Gateway).To(Equal([]string{ipv6Gateway})) + Expect(ipv6Spec.Ip).To(HaveLen(1)) + Expect(ipv6Spec.Ip[0]).To(BeAssignableToTypeOf(&types.CustomizationFixedIpV6{})) + addressSpec := ipv6Spec.Ip[0].(*types.CustomizationFixedIpV6) + Expect(addressSpec.IpAddress).To(Equal(ipv6)) + Expect(addressSpec.SubnetMask).To(BeEquivalentTo(ipv6Subnet)) + }) + }) + + Context("IPv4/6 DHCP", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + Name: "eth0", + DHCP4: true, + DHCP6: true, + MTU: 1500, // AFAIK not support via GOSC + Nameservers: []string{dnsServer1}, + Routes: nil, // AFAIK not support via GOSC + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(adapterMappings).To(HaveLen(1)) + mapping := adapterMappings[0] + + adapter := mapping.Adapter + Expect(mapping.MacAddress).To(Equal(macAddr1)) + Expect(adapter.Gateway).To(BeEmpty()) + Expect(adapter.SubnetMask).To(BeEmpty()) + + Expect(adapter.Ip).To(BeAssignableToTypeOf(&types.CustomizationDhcpIpGenerator{})) + ipv6Spec := adapter.IpV6Spec + Expect(ipv6Spec).ToNot(BeNil()) + Expect(ipv6Spec.Ip).To(HaveLen(1)) + Expect(ipv6Spec.Ip[0]).To(BeAssignableToTypeOf(&types.CustomizationDhcpIpV6Generator{})) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/netplan.go b/pkg/vmprovider/providers/vsphere2/network/netplan.go new file mode 100644 index 000000000..71ca80b45 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/netplan.go @@ -0,0 +1,103 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + "strings" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +// Netplan representation described in https://via.vmw.com/cloud-init-netplan // FIXME: 404. +type Netplan struct { + Version int `yaml:"version,omitempty"` + Ethernets map[string]NetplanEthernet `yaml:"ethernets,omitempty"` +} + +type NetplanEthernet struct { + Match NetplanEthernetMatch `yaml:"match,omitempty"` + SetName string `yaml:"set-name,omitempty"` + Dhcp4 bool `yaml:"dhcp4,omitempty"` + Dhcp6 bool `yaml:"dhcp6,omitempty"` + Addresses []string `yaml:"addresses,omitempty"` + Gateway4 string `yaml:"gateway4,omitempty"` + Gateway6 string `yaml:"gateway6,omitempty"` + MTU int64 `yaml:"mtu,omitempty"` + Nameservers NetplanEthernetNameserver `yaml:"nameservers,omitempty"` + Routes []NetplanEthernetRoute `yaml:"routes,omitempty"` +} + +type NetplanEthernetMatch struct { + MacAddress string `yaml:"macaddress,omitempty"` +} + +type NetplanEthernetNameserver struct { + Addresses []string `yaml:"addresses,omitempty"` + Search []string `yaml:"search,omitempty"` +} + +type NetplanEthernetRoute struct { + To string `yaml:"to"` + Via string `yaml:"via"` + Metric int32 `yaml:"metric,omitempty"` +} + +func NetPlanCustomization(result NetworkInterfaceResults) (*Netplan, error) { + netPlan := &Netplan{ + Version: constants.NetPlanVersion, + Ethernets: make(map[string]NetplanEthernet), + } + + for _, r := range result.Results { + npEth := NetplanEthernet{ + Match: NetplanEthernetMatch{ + MacAddress: NormalizeNetplanMac(r.MacAddress), + }, + SetName: r.Name, + MTU: r.MTU, + Nameservers: NetplanEthernetNameserver{ + Addresses: r.Nameservers, + Search: r.SearchDomains, + }, + } + + npEth.Dhcp4 = r.DHCP4 + npEth.Dhcp6 = r.DHCP6 + + if !npEth.Dhcp4 { + for _, ipConfig := range r.IPConfigs { + if ipConfig.IsIPv4 { + if npEth.Gateway4 == "" { + npEth.Gateway4 = ipConfig.Gateway + } + npEth.Addresses = append(npEth.Addresses, ipConfig.IPCIDR) + } + } + } + if !npEth.Dhcp6 { + for _, ipConfig := range r.IPConfigs { + if !ipConfig.IsIPv4 { + if npEth.Gateway6 == "" { + npEth.Gateway6 = ipConfig.Gateway + } + npEth.Addresses = append(npEth.Addresses, ipConfig.IPCIDR) + } + } + } + + for _, route := range r.Routes { + npEth.Routes = append(npEth.Routes, NetplanEthernetRoute(route)) + } + + netPlan.Ethernets[npEth.SetName] = npEth + } + + return netPlan, nil +} + +// NormalizeNetplanMac normalizes the mac address format to one compatible with netplan. +func NormalizeNetplanMac(mac string) string { + mac = strings.ReplaceAll(mac, "-", ":") + return strings.ToLower(mac) +} diff --git a/pkg/vmprovider/providers/vsphere2/network/netplan_test.go b/pkg/vmprovider/providers/vsphere2/network/netplan_test.go new file mode 100644 index 000000000..5e3e68375 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/netplan_test.go @@ -0,0 +1,143 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +var _ = Describe("Netplan", func() { + const ( + ifName = "eth0" + macAddr1 = "50-8A-80-9D-28-22" + macAddr1Norm = "50:8a:80:9d:28:22" + ipv4Gateway = "192.168.1.1" + ipv4 = "192.168.1.10" + ipv4CIDR = ipv4 + "/24" + ipv6Gateway = "fd8e:b5a0:f172:123::1" + ipv6 = "fd8e:b5a0:f172:123::f" + ipv6Subnet = 48 + dnsServer1 = "9.9.9.9" + searchDomain1 = "foobar.local" + ) + + Context("NetPlanCustomization", func() { + + var ( + results network.NetworkInterfaceResults + netplan *network.Netplan + err error + ) + + BeforeEach(func() { + results.Results = nil + netplan = nil + }) + + JustBeforeEach(func() { + netplan, err = network.NetPlanCustomization(results) + }) + + Context("IPv4/6 Static adapter", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + IPCIDR: ipv4CIDR, + IsIPv4: true, + Gateway: ipv4Gateway, + }, + { + IPCIDR: ipv6 + fmt.Sprintf("/%d", ipv6Subnet), + IsIPv4: false, + Gateway: ipv6Gateway, + }, + }, + MacAddress: macAddr1, + Name: ifName, + DHCP4: false, + DHCP6: false, + MTU: 1500, + Nameservers: []string{dnsServer1}, + SearchDomains: []string{searchDomain1}, + Routes: []network.NetworkInterfaceRoute{ + { + To: "185.107.56.59", + Via: "10.1.1.1", + Metric: 42, + }, + }, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(netplan).ToNot(BeNil()) + Expect(netplan.Version).To(Equal(constants.NetPlanVersion)) + + Expect(netplan.Ethernets).To(HaveLen(1)) + Expect(netplan.Ethernets).To(HaveKey(ifName)) + + np := netplan.Ethernets[ifName] + Expect(np.Match.MacAddress).To(Equal(macAddr1Norm)) + Expect(np.SetName).To(Equal(ifName)) + Expect(np.Dhcp4).To(BeFalse()) + Expect(np.Dhcp6).To(BeFalse()) + Expect(np.Gateway4).To(Equal(ipv4Gateway)) + Expect(np.Gateway6).To(Equal(ipv6Gateway)) + Expect(np.MTU).To(BeEquivalentTo(1500)) + Expect(np.Nameservers.Addresses).To(Equal([]string{dnsServer1})) + Expect(np.Nameservers.Search).To(Equal([]string{searchDomain1})) + Expect(np.Routes).To(HaveLen(1)) + route := np.Routes[0] + Expect(route.To).To(Equal("185.107.56.59")) + Expect(route.Via).To(Equal("10.1.1.1")) + Expect(route.Metric).To(BeEquivalentTo(42)) + }) + }) + + Context("IPv4/6 DHCP", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + Name: "eth0", + DHCP4: true, + DHCP6: true, + MTU: 9000, + Nameservers: []string{dnsServer1}, + SearchDomains: []string{searchDomain1}, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(netplan).ToNot(BeNil()) + Expect(netplan.Version).To(Equal(constants.NetPlanVersion)) + + Expect(netplan.Ethernets).To(HaveLen(1)) + Expect(netplan.Ethernets).To(HaveKey(ifName)) + + np := netplan.Ethernets[ifName] + Expect(np.Match.MacAddress).To(Equal(macAddr1Norm)) + Expect(np.SetName).To(Equal(ifName)) + Expect(np.Dhcp4).To(BeTrue()) + Expect(np.Dhcp6).To(BeTrue()) + Expect(np.MTU).To(BeEquivalentTo(9000)) + Expect(np.Nameservers.Addresses).To(Equal([]string{dnsServer1})) + Expect(np.Nameservers.Search).To(Equal([]string{searchDomain1})) + Expect(np.Routes).To(BeEmpty()) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/network.go b/pkg/vmprovider/providers/vsphere2/network/network.go new file mode 100644 index 000000000..e5d5d96e5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network.go @@ -0,0 +1,630 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//nolint:revive +package network + +import ( + goctx "context" + "fmt" + "net" + "strings" + "time" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + vimtypes "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type NetworkInterfaceResults struct { + Results []NetworkInterfaceResult +} + +type NetworkInterfaceResult struct { + IPConfigs []NetworkInterfaceIPConfig + MacAddress string + ExternalID string + NetworkID string + Backing object.NetworkReference + + Device vimtypes.BaseVirtualDevice + + // Fields from the InterfaceSpec used later during customization. + Name string + DHCP4 bool + DHCP6 bool + MTU int64 + Nameservers []string + SearchDomains []string + Routes []NetworkInterfaceRoute +} + +type NetworkInterfaceIPConfig struct { + IPCIDR string // IP address in CIDR notation e.g. 192.168.10.42/24 + IsIPv4 bool + Gateway string +} + +type NetworkInterfaceRoute struct { + To string + Via string + Metric int32 +} + +const ( + retryInterval = 100 * time.Millisecond + defaultEthernetCardType = "vmxnet3" +) + +var ( + // RetryTimeout is var so tests can change it to shorten tests until we get rid of the poll. + RetryTimeout = 15 * time.Second +) + +// CreateAndWaitForNetworkInterfaces creates the appropriate CRs for the VM's network +// interfaces, and then waits for them to be reconciled by NCP (NSX-T) or NetOP (VDS). +// +// Networking has always been kind of a pain and clunky for us, and unfortunately this +// code suffers gotchas and other not-so-great limitations. +// +// - Historically, this code used wait.PollImmediate() and we continue to do so here, +// but eventually we should Watch() these resources. Note though, that in the very +// common case, the CR is reconciled before our poll timeout, so that does save us +// from bailing out of the Reconcile. +// - Both NCP and NetOP CR Status inform us of the backing and IPAM info. However, for +// our InterfaceSpec we allow for DHCP but neither NCP nor NetOP has a way for us to +// mark the CR to don't do IPAM or to check DHCP is even enabled on the network. So +// this burns an IP, and the user must know that DHCP is actually configured. +// - CR naming has mostly been working by luck, and sometimes didn't offer very good +// discoverability. Here, with v1a2 we now have a "name" field in our InterfaceSpec, +// so we use that (BMV: need to double-check that field meets k8s name requirements) +// A longer term option is to use GenerateName to ensure a unique name, and then +// client.List() and filter by the OwnerRef to find the VM's network CRs, and to +// annotate the CRs to help identify which VM InterfaceSpec it corresponds to. +// Note that for existing v1a1 VMs we may need to add legacy name support here to +// find their interface CRs. +// - Instead of CreateOrUpdate, use CreateOrPatch to lessen the odds of blowing away +// any new fields. +func CreateAndWaitForNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + finder *find.Finder, + clusterMoRef *vimtypes.ManagedObjectReference, + interfaces []vmopv1.VirtualMachineNetworkInterfaceSpec) (NetworkInterfaceResults, error) { + + networkType := lib.GetNetworkProviderType() + if networkType == "" { + return NetworkInterfaceResults{}, fmt.Errorf("no network provider set") + } + + results := make([]NetworkInterfaceResult, 0, len(interfaces)) + + for i := range interfaces { + interfaceSpec := &interfaces[i] + + var result *NetworkInterfaceResult + var err error + + switch networkType { + case lib.NetworkProviderTypeVDS: + result, err = createNetOPNetworkInterface(vmCtx, client, vimClient, interfaceSpec) + case lib.NetworkProviderTypeNSXT: + result, err = createNCPNetworkInterface(vmCtx, client, vimClient, clusterMoRef, interfaceSpec) + case lib.NetworkProviderTypeNamed: + result, err = createNamedNetworkInterface(vmCtx, finder, interfaceSpec) + default: + err = fmt.Errorf("unsupported network provider envvar value: %q", networkType) + } + + if err != nil { + return NetworkInterfaceResults{}, + fmt.Errorf("network interface %q error: %w", interfaceSpec.Name, err) + } + + applyInterfaceSpecToResult(interfaceSpec, result) + results = append(results, *result) + } + + // TODO: Once we really support network changing on the fly, we need to keep track of now + // unused network interface CRDs so they can be deleted after they're removed from the VM + // via Reconfigure, instead of delaying that until the VM is deleted via GC. + + return NetworkInterfaceResults{ + Results: results, + }, nil +} + +// applyInterfaceSpecToResult applies the InterfaceSpec to results. Much of the InterfaceSpec - like DHCP - +// cannot be specified to the underlying network provider so apply those overrides to the results. +func applyInterfaceSpecToResult( + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec, + result *NetworkInterfaceResult) { + + // We don't really support IPv6 yet so don't enable it when the underlying provider didn't return any IPs. + dhcp4 := interfaceSpec.DHCP4 || len(result.IPConfigs) == 0 + dhcp6 := interfaceSpec.DHCP6 + + if len(interfaceSpec.Addresses) > 0 { + // The InterfaceSpec takes precedence over what underlying network provider says, so in this case it + // likely it did IPAM but we'll ignore those IPs. Providing static IPs via the Addresses field is + // probably not very common so override the IPConfigs so bootstrap has only one field to use. + result.IPConfigs = make([]NetworkInterfaceIPConfig, 0, len(interfaceSpec.Addresses)) + + for _, addr := range interfaceSpec.Addresses { + ip, _, err := net.ParseCIDR(addr) + if err != nil { + continue + } + + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: addr, + IsIPv4: ip.To4() != nil, + } + + if ipConfig.IsIPv4 { + dhcp4 = false + ipConfig.Gateway = interfaceSpec.Gateway4 + } else { + dhcp6 = false + ipConfig.Gateway = interfaceSpec.Gateway6 + } + + result.IPConfigs = append(result.IPConfigs, ipConfig) + } + } + + result.Name = interfaceSpec.Name + result.DHCP4 = dhcp4 + result.DHCP6 = dhcp6 + result.Nameservers = interfaceSpec.Nameservers + result.SearchDomains = interfaceSpec.SearchDomains + + if interfaceSpec.MTU != nil { + result.MTU = *interfaceSpec.MTU + } + for _, route := range interfaceSpec.Routes { + result.Routes = append(result.Routes, NetworkInterfaceRoute{To: route.To, Via: route.Via, Metric: route.Metric}) + } +} + +func createNamedNetworkInterface( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + if interfaceSpec.Network.Kind != "" || interfaceSpec.Network.APIVersion != "" { + return nil, fmt.Errorf("network TypeMeta not supported for name network: %v", interfaceSpec.Network.TypeMeta) + } + + networkName := interfaceSpec.Network.Name + if networkName == "" { + return nil, fmt.Errorf("network name is required") + } + + backing, err := finder.Network(vmCtx, networkName) + if err != nil { + return nil, fmt.Errorf("unable to find named network %q: %w", networkName, err) + } + + return &NetworkInterfaceResult{ + NetworkID: networkName, + Backing: backing, + }, nil +} + +// NetOPCRName returns the name to be used for the NetOP NetworkInterface CR. +func NetOPCRName(vmName, networkName, interfaceName string, isV1A1 bool) string { + var name string + + if isV1A1 { + // Old naming convention: each network can really only have 1 NIC. + if networkName != "" { + name = fmt.Sprintf("%s-%s", networkName, vmName) + } else { + name = vmName + } + } else { + if networkName != "" { + name = fmt.Sprintf("%s-%s-%s", vmName, networkName, interfaceName) + } else { + name = fmt.Sprintf("%s-%s", vmName, interfaceName) + } + } + + return name +} + +func createNetOPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + if kind := interfaceSpec.Network.Kind; kind != "" && kind != "Network" { + return nil, fmt.Errorf("network kind %q is not supported for VDS", kind) + } + + // If empty, NetOP will try to select a namespace default. + networkName := interfaceSpec.Network.Name + + netIf := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: NetOPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), + Namespace: vmCtx.VM.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(vmCtx, client, netIf, func() error { + if err := controllerutil.SetOwnerReference(vmCtx.VM, netIf, client.Scheme()); err != nil { + // If this fails we likely have an object name collision, and we're in a tough spot. + return err + } + + netIf.Spec.NetworkName = networkName + // NetOP only defines a VMXNet3 type, but it doesn't really matter for our purposes. + netIf.Spec.Type = netopv1alpha1.NetworkInterfaceTypeVMXNet3 + return nil + }) + + if err != nil { + return nil, err + } + + netIf, err = waitForReadyNetworkInterface(vmCtx, client, netIf.Name) + if err != nil { + return nil, err + } + + return netOpNetIfToResult(vimClient, netIf), nil +} + +func netOpNetIfToResult( + vimClient *vim25.Client, + netIf *netopv1alpha1.NetworkInterface) *NetworkInterfaceResult { + + ipConfigs := make([]NetworkInterfaceIPConfig, 0, len(netIf.Status.IPConfigs)) + for _, ip := range netIf.Status.IPConfigs { + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: ipCIDRNotation(ip.IP, ip.SubnetMask, ip.IPFamily == netopv1alpha1.IPv4Protocol), + IsIPv4: ip.IPFamily == netopv1alpha1.IPv4Protocol, + Gateway: ip.Gateway, + } + ipConfigs = append(ipConfigs, ipConfig) + } + + pgObjRef := vimtypes.ManagedObjectReference{ + Type: "DistributedVirtualPortgroup", + Value: netIf.Status.NetworkID, + } + + return &NetworkInterfaceResult{ + IPConfigs: ipConfigs, + MacAddress: netIf.Status.MacAddress, // Not set by NetOP. + ExternalID: netIf.Status.ExternalID, // Ditto. + NetworkID: netIf.Status.NetworkID, + Backing: object.NewDistributedVirtualPortgroup(vimClient, pgObjRef), + } +} + +func findNetOPCondition( + netIf *netopv1alpha1.NetworkInterface, + condType netopv1alpha1.NetworkInterfaceConditionType) *netopv1alpha1.NetworkInterfaceCondition { + + for i := range netIf.Status.Conditions { + if netIf.Status.Conditions[i].Type == condType { + return &netIf.Status.Conditions[i] + } + } + return nil +} + +func waitForReadyNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + name string) (*netopv1alpha1.NetworkInterface, error) { + + netIf := &netopv1alpha1.NetworkInterface{} + netIfKey := types.NamespacedName{Namespace: vmCtx.VM.Namespace, Name: name} + + // TODO: Watch() this type instead. + err := wait.PollUntilContextTimeout(vmCtx, retryInterval, RetryTimeout, true, func(_ goctx.Context) (bool, error) { + if err := client.Get(vmCtx, netIfKey, netIf); err != nil { + return false, ctrlruntime.IgnoreNotFound(err) + } + + cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceReady) + return cond != nil && cond.Status == corev1.ConditionTrue, nil + }) + + if err != nil { + if wait.Interrupted(err) { + // Try to return a more meaningful error when timed out. + if cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceFailure); cond != nil && cond.Status == corev1.ConditionTrue { + return nil, fmt.Errorf("network interface failure: %s - %s", cond.Reason, cond.Message) + } + if cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceReady); cond != nil && cond.Status == corev1.ConditionFalse { + return nil, fmt.Errorf("network interface is not ready: %s - %s", cond.Reason, cond.Message) + } + return nil, fmt.Errorf("network interface is not ready yet") + } + + return nil, err + } + + return netIf, nil +} + +// NCPCRName returns the name to be used for the NCP VirtualNetworkInterface CR. +func NCPCRName(vmName, networkName, interfaceName string, isV1A1 bool) string { + var name string + + if isV1A1 { + name = fmt.Sprintf("%s-lsp", vmName) + if networkName != "" { + name = fmt.Sprintf("%s-%s", networkName, name) + } + + } else { + if networkName != "" { + name = fmt.Sprintf("%s-%s-%s", vmName, networkName, interfaceName) + } else { + name = fmt.Sprintf("%s-%s", vmName, interfaceName) + } + } + + return name +} + +func createNCPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + clusterMoRef *vimtypes.ManagedObjectReference, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + // TODO: Do we need to still support the odd-ball NetOP in NSX-T? Sigh. Do that check here if needed. + if kind := interfaceSpec.Network.Kind; kind != "" && kind != "VirtualNetwork" { + return nil, fmt.Errorf("network kind %q is not supported for NCP", kind) + } + + // If empty, NCP will use the namespace default. + networkName := interfaceSpec.Network.Name + + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: NCPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), + Namespace: vmCtx.VM.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(vmCtx, client, vnetIf, func() error { + if err := controllerutil.SetOwnerReference(vmCtx.VM, vnetIf, client.Scheme()); err != nil { + return err + } + + vnetIf.Spec.VirtualNetwork = networkName + return nil + }) + + if err != nil { + return nil, err + } + + vnetIf, err = waitForReadyNCPNetworkInterface(vmCtx, client, vnetIf.Name) + if err != nil { + return nil, err + } + + return ncpNetIfToResult(vmCtx, vimClient, clusterMoRef, vnetIf) +} + +func ncpNetIfToResult( + ctx goctx.Context, + vimClient *vim25.Client, + clusterMoRef *vimtypes.ManagedObjectReference, + vnetIf *ncpv1alpha1.VirtualNetworkInterface) (*NetworkInterfaceResult, error) { + + // NSX-T makes the backing determination difficult: NsxLogicalSwitchID must be mapped to an + // actual DVPG since that is the backing, but the DVPG can, in some very rare but supported + // configurations, vary between CCRs. If we know the CCR - ether the VM already exists, or + // (for later work) we might pre-determine CCR w/o placement if there is only one possibility - + // get that backing now. + // Otherwise, we'll do it post-placement via ResolveNCPBackingPostPlacement() so that we create + // the VM with the correct backing. That means we cannot make this a part of the PlaceVMxCluster() + // ConfigSpec since we don't know the backing: we'd have to pre-filter the placement candidates. + // What a mess. This is an unfortunate decision that forces mapping logic to every NCP consumer. + + var backing object.NetworkReference + networkID := vnetIf.Status.ProviderStatus.NsxLogicalSwitchID + + if clusterMoRef != nil { + ccr := object.NewClusterComputeResource(vimClient, *clusterMoRef) + + networkRef, err := searchNsxtNetworkReference(ctx, ccr, networkID) + if err != nil { + return nil, err + } + + backing = networkRef + } + + var ipConfigs []NetworkInterfaceIPConfig + if ipAddress := vnetIf.Status.IPAddresses; len(ipAddress) == 0 || (len(ipAddress) == 1 && ipAddress[0].IP == "") { + // NCP's way of saying DHCP. + } else { + // Historically, we only grabbed the first entry and assume it is always IPv4 (!!!). Try to do slightly better. + for _, ipAddr := range ipAddress { + if ipAddr.IP == "" { + continue + } + + isIPv4 := net.ParseIP(ipAddr.IP).To4() != nil + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: ipCIDRNotation(ipAddr.IP, ipAddr.SubnetMask, isIPv4), + IsIPv4: isIPv4, + Gateway: ipAddr.Gateway, + } + + ipConfigs = append(ipConfigs, ipConfig) + } + } + + result := &NetworkInterfaceResult{ + IPConfigs: ipConfigs, + MacAddress: vnetIf.Status.MacAddress, + ExternalID: vnetIf.Status.InterfaceID, + NetworkID: networkID, + Backing: backing, + } + + return result, nil +} + +func waitForReadyNCPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + name string) (*ncpv1alpha1.VirtualNetworkInterface, error) { + + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{} + vnetIfKey := types.NamespacedName{Namespace: vmCtx.VM.Namespace, Name: name} + + // TODO: Watch() this type instead. + err := wait.PollUntilContextTimeout(vmCtx, retryInterval, RetryTimeout, true, func(_ goctx.Context) (bool, error) { + if err := client.Get(vmCtx, vnetIfKey, vnetIf); err != nil { + return false, ctrlruntime.IgnoreNotFound(err) + } + + for _, condition := range vnetIf.Status.Conditions { + // TODO: Does NCP define condition constants? + if strings.Contains(condition.Type, "Ready") && strings.Contains(condition.Status, "True") { + return true, nil + } + } + + return false, nil + }) + + if err != nil { + if wait.Interrupted(err) { + // Try to return a more meaningful error when timed out. + for _, cond := range vnetIf.Status.Conditions { + if strings.Contains(cond.Type, "Ready") && !strings.Contains(cond.Status, "True") { + return nil, fmt.Errorf("network interface is not ready: %s - %s", cond.Reason, cond.Message) + } + } + // TODO: NCP also has an annotation but that usually doesn't provide very useful details. + return nil, fmt.Errorf("network interface is not ready yet") + } + + return nil, err + } + + if vnetIf.Status.ProviderStatus == nil { + return nil, fmt.Errorf("network interface is ready but does not have provider status") + } + + return vnetIf, nil +} + +// ipCIDRNotation takes the IP and subnet mask and returns the IP in CIDR notation. +// TODO: Better error checking. Nail down exactly how we want handle IPv4inV6 addresses. +func ipCIDRNotation(ip string, mask string, isIPv4 bool) string { + if isIPv4 { + ipNet := net.IPNet{ + IP: net.ParseIP(ip).To4(), + Mask: net.IPMask(net.ParseIP(mask).To4()), + } + return ipNet.String() + } + + ipNet := net.IPNet{ + IP: net.ParseIP(ip).To16(), + Mask: net.IPMask(net.ParseIP(mask).To16()), + } + + return ipNet.String() +} + +// CreateDefaultEthCard creates a default Ethernet card attached to the backing. This is used +// when the VM Class ConfigSpec does not have a device entry for a VM Spec network interface, +// so we need a new device. +func CreateDefaultEthCard( + ctx goctx.Context, + result *NetworkInterfaceResult) (vimtypes.BaseVirtualDevice, error) { + + // We may not have the backing yet if this is NSX-T. The backing will be resolved after placement + // when we'll know the CCR, so we can resolve the correct DVPG. + if result.Backing == nil { + return nil, nil + } + + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + if err != nil { + return nil, fmt.Errorf("unable to get ethernet card backing info for network %v: %w", result.Backing.Reference(), err) + } + + dev, err := object.EthernetCardTypes().CreateEthernetCard(defaultEthernetCardType, backing) + if err != nil { + return nil, fmt.Errorf("unable to create ethernet card network %v: %w", result.Backing.Reference(), err) + } + + ethCard := dev.(vimtypes.BaseVirtualEthernetCard).GetVirtualEthernetCard() + ethCard.ExternalId = result.ExternalID + if result.MacAddress != "" { + ethCard.MacAddress = result.MacAddress + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeManual) + } else { + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeGenerated) // TODO: Or TypeAssigned? + } + + return dev, nil +} + +// ApplyInterfaceResultToVirtualEthCard applies the interface result from the NetOP/NCP +// provider to an existing Ethernet device from the class ConfigSpec. +func ApplyInterfaceResultToVirtualEthCard( + ctx goctx.Context, + ethCard *vimtypes.VirtualEthernetCard, + result *NetworkInterfaceResult) error { + + ethCard.ExternalId = result.ExternalID + if result.MacAddress != "" { + // BMV: Too much confusion and possible breakage if we don't honor the provider MAC. + // Otherwise, IMO a foot gun and will break on setups that enforce MAC filtering. + ethCard.MacAddress = result.MacAddress + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeManual) + } else { //nolint + // BMV: IMO this must be Generated/TypeAssigned to avoid major foot gun, but we have tests assuming + // this is left as-is. + // We should have a MAC address field to the VM.Spec if we want this to be specified by the user. + // ethCard.MacAddress = "" + // ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeGenerated) + } + + // We may not have the backing yet if this is NSX-T. The backing will be resolved after placement + // when we'll know the CCR, so we can resolve the correct DVPG. + if result.Backing != nil { + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + if err != nil { + return fmt.Errorf("unable to get ethernet card backing info for network %v: %w", result.NetworkID, err) + } + ethCard.Backing = backing + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go b/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go new file mode 100644 index 000000000..29fcd237c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestNetwork(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Network Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/network/network_test.go b/pkg/vmprovider/providers/vsphere2/network/network_test.go new file mode 100644 index 000000000..c57a5e1ab --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network_test.go @@ -0,0 +1,344 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + goctx "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + interfaceSpecs []vmopv1.VirtualMachineNetworkInterfaceSpec + + results network.NetworkInterfaceResults + err error + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "network-test-vm", + Namespace: "network-test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithName("network_test"), + VM: vm, + } + + interfaceSpecs = nil + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Context("Named Network", func() { + // Use network vcsim automatically creates. + const networkName = "DC0_DVPG0" + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + }) + + Context("network exists", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: networkName}, + DHCP6: true, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(1)) + + result := results.Results[0] + By("has expected backing", func() { + Expect(result.Backing).ToNot(BeNil()) + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + Expect(err).ToNot(HaveOccurred()) + backingInfo, ok := backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).To(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + + Expect(result.DHCP4).To(BeTrue()) + Expect(result.DHCP6).To(BeTrue()) // Only enabled if explicitly requested (which it is above). + }) + }) + + Context("network does not exist", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: "bogus"}, + }, + } + }) + + It("returns error", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unable to find named network")) + Expect(results.Results).To(BeEmpty()) + }) + }) + }) + + Context("VDS", func() { + const ( + interfaceName = "eth0" + networkName = "my-vds-network" + ) + + BeforeEach(func() { + network.RetryTimeout = 1 * time.Second + testConfig.WithNetworkEnv = builder.NetworkEnvVDS + }) + + Context("Simulate workflow", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + } + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NetOP reconcile", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.NetworkName).To(Equal(networkName)) + + netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value + netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.110", + IPFamily: netopv1alpha1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + IPFamily: netopv1alpha1.IPv6Protocol, + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(BeEmpty()) + Expect(result.ExternalID).To(BeEmpty()) + Expect(result.NetworkID).To(Equal(ctx.NetworkRef.Reference().Value)) + Expect(result.Backing).ToNot(BeNil()) + Expect(result.Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + }) + }) + }) + + Context("NCP", func() { + const ( + interfaceName = "eth0" + interfaceID = "my-interface-id" + networkName = "my-ncp-network" + macAddress = "01-23-45-67-89-AB-CD-EF" + ) + + BeforeEach(func() { + network.RetryTimeout = 1 * time.Second + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + }) + + Context("Simulate workflow", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + } + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NCP reconcile", func() { + netInterface := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.VirtualNetwork).To(Equal(networkName)) + + netInterface.Status.InterfaceID = interfaceID + netInterface.Status.MacAddress = macAddress + netInterface.Status.ProviderStatus = &ncpv1alpha1.VirtualNetworkInterfaceProviderStatus{ + NsxLogicalSwitchID: builder.NsxTLogicalSwitchUUID, + } + netInterface.Status.IPAddresses = []ncpv1alpha1.VirtualNetworkInterfaceIP{ + { + IP: "192.168.1.110", + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []ncpv1alpha1.VirtualNetworkCondition{ + { + Type: "Ready", + Status: "True", + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(Equal(macAddress)) + Expect(result.ExternalID).To(Equal(interfaceID)) + Expect(result.NetworkID).To(Equal(builder.NsxTLogicalSwitchUUID)) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + + // Without the ClusterMoRef on the first call this will be nil for NSXT. + Expect(result.Backing).To(BeNil()) + + clusterMoRef := ctx.GetSingleClusterCompute().Reference() + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + &clusterMoRef, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(1)) + Expect(results.Results[0].Backing).ToNot(BeNil()) + Expect(results.Results[0].Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/nsxt.go b/pkg/vmprovider/providers/vsphere2/network/nsxt.go new file mode 100644 index 000000000..d08c6a6b1 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/nsxt.go @@ -0,0 +1,117 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + goctx "context" + "fmt" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +// ResolveBackingPostPlacement fixes up the backings where we did not know the CCR until after +// placement. This should be called if CreateAndWaitForNetworkInterfaces() was called with a nil +// clusterMoRef. Returns true if a backing was resolved, so the ConfigSpec needs to be updated. +func ResolveBackingPostPlacement( + ctx goctx.Context, + vimClient *vim25.Client, + clusterMoRef vimtypes.ManagedObjectReference, + results *NetworkInterfaceResults) (bool, error) { + + if len(results.Results) == 0 { + return false, nil + } + + networkType := lib.GetNetworkProviderType() + if networkType == "" { + return false, fmt.Errorf("no network provider set") + } + + ccr := object.NewClusterComputeResource(vimClient, clusterMoRef) + fixedUp := false + + for idx := range results.Results { + if results.Results[idx].Backing != nil { + continue + } + + var backing object.NetworkReference + var err error + + switch networkType { + case lib.NetworkProviderTypeNSXT: + backing, err = searchNsxtNetworkReference(ctx, ccr, results.Results[idx].NetworkID) + if err != nil { + err = fmt.Errorf("post placement NSX-T backing fixup failed: %w", err) + } + default: + err = fmt.Errorf("only NSX-T networks are expected to need post placement backing fixup") + } + + if err != nil { + return false, err + } + + fixedUp = true + results.Results[idx].Backing = backing + } + + return fixedUp, nil +} + +// searchNsxtNetworkReference takes in NSX-T LogicalSwitchUUID and returns the reference of the network. +func searchNsxtNetworkReference( + ctx goctx.Context, + ccr *object.ClusterComputeResource, + networkID string) (object.NetworkReference, error) { + + // This is more or less how the old code did it. We could save repeated work by moving this + // into the callers since it will always be for the same CCR, but the common case is one NIC, + // or at most a handful, so that's for later. + var obj mo.ClusterComputeResource + if err := ccr.Properties(ctx, ccr.Reference(), []string{"network"}, &obj); err != nil { + return nil, err + } + + var dvpgsMoRefs []vimtypes.ManagedObjectReference + for _, n := range obj.Network { + if n.Type == "DistributedVirtualPortgroup" { + dvpgsMoRefs = append(dvpgsMoRefs, n.Reference()) + } + } + + if len(dvpgsMoRefs) == 0 { + return nil, fmt.Errorf("ClusterComputeResource %s has no DVPGs", ccr.Reference().Value) + } + + var dvpgs []mo.DistributedVirtualPortgroup + err := property.DefaultCollector(ccr.Client()).Retrieve(ctx, dvpgsMoRefs, []string{"config.logicalSwitchUuid"}, &dvpgs) + if err != nil { + return nil, err + } + + var dvpgMoRefs []vimtypes.ManagedObjectReference + for _, dvpg := range dvpgs { + if dvpg.Config.LogicalSwitchUuid == networkID { + dvpgMoRefs = append(dvpgMoRefs, dvpg.Reference()) + } + } + + switch len(dvpgMoRefs) { + case 1: + return object.NewDistributedVirtualPortgroup(ccr.Client(), dvpgMoRefs[0]), nil + case 0: + return nil, fmt.Errorf("no DVPG with NSX-T network ID %q found", networkID) + default: + // The LogicalSwitchUuid is supposed to be unique per CCR, so this is likely an NCP + // misconfiguration, and we don't know which one to pick. + return nil, fmt.Errorf("multiple DVPGs (%d) with NSX-T network ID %q found", len(dvpgMoRefs), networkID) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go b/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go new file mode 100644 index 000000000..35925553a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("ResolveBackingPostPlacement", func() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + + results *network.NetworkInterfaceResults + fixedUp bool + err error + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + fixedUp, err = network.ResolveBackingPostPlacement( + ctx, + ctx.VCClient.Client, + ctx.GetSingleClusterCompute().Reference(), + results) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Context("returns success", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + + results = &network.NetworkInterfaceResults{ + Results: []network.NetworkInterfaceResult{ + { + NetworkID: builder.NsxTLogicalSwitchUUID, + Backing: nil, + }, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(fixedUp).To(BeTrue()) + + Expect(results.Results).To(HaveLen(1)) + By("should populate the backing", func() { + backing := results.Results[0].Backing + Expect(backing).ToNot(BeNil()) + Expect(backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go new file mode 100644 index 000000000..6a28e8a12 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go @@ -0,0 +1,206 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + goctx "context" + "fmt" + "strings" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +// Recommendation is the info about a placement recommendation. +type Recommendation struct { + PoolMoRef types.ManagedObjectReference + HostMoRef *types.ManagedObjectReference + // TODO: Datastore, whatever else as we need it. +} + +func relocateSpecToRecommendation(relocateSpec *types.VirtualMachineRelocateSpec) *Recommendation { + // Instance Storage requires the host. + if relocateSpec == nil || relocateSpec.Pool == nil || relocateSpec.Host == nil { + return nil + } + + return &Recommendation{ + PoolMoRef: *relocateSpec.Pool, + HostMoRef: relocateSpec.Host, + } +} + +func clusterPlacementActionToRecommendation(action types.ClusterClusterInitialPlacementAction) *Recommendation { + return &Recommendation{ + PoolMoRef: action.Pool, + HostMoRef: action.TargetHost, + } +} + +func CheckPlacementRelocateSpec(spec *types.VirtualMachineRelocateSpec) error { + if spec == nil { + return fmt.Errorf("RelocateSpec is nil") + } + if spec.Host == nil { + return fmt.Errorf("RelocateSpec does not have a host") + } + if spec.Pool == nil { + return fmt.Errorf("RelocateSpec does not have a resource pool") + } + if spec.Datastore == nil { + return fmt.Errorf("RelocateSpec does not have a datastore") + } + return nil +} + +func ParseRelocateVMResponse( + vmCtx context.VirtualMachineContextA2, + res *types.PlacementResult) *types.VirtualMachineRelocateSpec { + + for _, r := range res.Recommendations { + if r.Reason == string(types.RecommendationReasonCodeXvmotionPlacement) { + for _, a := range r.Action { + if pa, ok := a.(*types.PlacementAction); ok { + if err := CheckPlacementRelocateSpec(pa.RelocateSpec); err != nil { + vmCtx.Logger.V(6).Info("Skipped RelocateSpec", + "reason", err.Error(), "relocateSpec", pa.RelocateSpec) + continue + } + + return pa.RelocateSpec + } + } + } + } + + return nil +} + +func CloneVMRelocateSpec( + vmCtx context.VirtualMachineContextA2, + cluster *object.ClusterComputeResource, + vmRef types.ManagedObjectReference, + cloneSpec *types.VirtualMachineCloneSpec) (*types.VirtualMachineRelocateSpec, error) { + + placementSpec := types.PlacementSpec{ + PlacementType: string(types.PlacementSpecPlacementTypeClone), + CloneSpec: cloneSpec, + RelocateSpec: &cloneSpec.Location, + CloneName: cloneSpec.Config.Name, + Vm: &vmRef, + } + + resp, err := cluster.PlaceVm(vmCtx, placementSpec) + if err != nil { + return nil, err + } + + rSpec := ParseRelocateVMResponse(vmCtx, resp) + if rSpec == nil { + return nil, fmt.Errorf("no valid placement action") + } + + return rSpec, nil +} + +// PlaceVMForCreate determines the suitable placement candidates in the cluster. +func PlaceVMForCreate( + ctx goctx.Context, + cluster *object.ClusterComputeResource, + configSpec *types.VirtualMachineConfigSpec) ([]Recommendation, error) { + + placementSpec := types.PlacementSpec{ + PlacementType: string(types.PlacementSpecPlacementTypeCreate), + ConfigSpec: configSpec, + } + + resp, err := cluster.PlaceVm(ctx, placementSpec) + if err != nil { + return nil, err + } + + var recommendations []Recommendation + + for _, r := range resp.Recommendations { + if r.Reason != string(types.RecommendationReasonCodeXvmotionPlacement) { + continue + } + + for _, a := range r.Action { + if pa, ok := a.(*types.PlacementAction); ok { + if r := relocateSpecToRecommendation(pa.RelocateSpec); r != nil { + recommendations = append(recommendations, *r) + } + } + } + } + + return recommendations, nil +} + +// ClusterPlaceVMForCreate determines the suitable cluster placement among the specified ResourcePools. +func ClusterPlaceVMForCreate( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + resourcePoolsMoRefs []types.ManagedObjectReference, + configSpec *types.VirtualMachineConfigSpec, + needsHost bool) ([]Recommendation, error) { + + // Work around PlaceVmsXCluster bug that crashes vpxd when ConfigSpec.Files is nil. + cs := *configSpec + cs.Files = new(types.VirtualMachineFileInfo) + + placementSpec := types.PlaceVmsXClusterSpec{ + ResourcePools: resourcePoolsMoRefs, + VmPlacementSpecs: []types.PlaceVmsXClusterSpecVmPlacementSpec{ + { + ConfigSpec: cs, + }, + }, + HostRecommRequired: &needsHost, + } + + vmCtx.Logger.V(6).Info("PlaceVmxCluster request", "placementSpec", placementSpec) + + resp, err := object.NewRootFolder(vcClient).PlaceVmsXCluster(vmCtx, placementSpec) + if err != nil { + return nil, err + } + + vmCtx.Logger.V(6).Info("PlaceVmxCluster response", "resp", resp) + + if len(resp.Faults) != 0 { + var faultMgs []string + for _, f := range resp.Faults { + msgs := make([]string, 0, len(f.Faults)) + for _, ff := range f.Faults { + msgs = append(msgs, ff.LocalizedMessage) + } + faultMgs = append(faultMgs, + fmt.Sprintf("ResourcePool %s faults: %s", f.ResourcePool.Value, strings.Join(msgs, ", "))) + } + return nil, fmt.Errorf("PlaceVmsXCluster faults: %v", faultMgs) + } + + var recommendations []Recommendation + + for _, info := range resp.PlacementInfos { + if info.Recommendation.Reason != string(types.RecommendationReasonCodeXClusterPlacement) { + continue + } + + for _, a := range info.Recommendation.Action { + if ca, ok := a.(*types.ClusterClusterInitialPlacementAction); ok { + if r := clusterPlacementActionToRecommendation(*ca); r != nil { + recommendations = append(recommendations, *r) + } + } + } + } + + return recommendations, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go new file mode 100644 index 000000000..8dc78a2a8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go @@ -0,0 +1,136 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/placement" +) + +func createRelocateSpec() *types.VirtualMachineRelocateSpec { + spec := &types.VirtualMachineRelocateSpec{} + spec.Host = &types.ManagedObjectReference{} + spec.Pool = &types.ManagedObjectReference{} + spec.Datastore = &types.ManagedObjectReference{} + return spec +} + +func createValidPlacementAction() (types.BaseClusterAction, *types.VirtualMachineRelocateSpec) { + action := types.PlacementAction{} + action.RelocateSpec = createRelocateSpec() + return types.BaseClusterAction(&action), action.RelocateSpec +} + +func createInvalidPlacementAction() types.BaseClusterAction { + action := types.PlacementAction{} + action.RelocateSpec = createRelocateSpec() + action.RelocateSpec.Host = nil + return types.BaseClusterAction(&action) +} + +func createStoragePlacementAction() types.BaseClusterAction { + action := types.StoragePlacementAction{} + action.RelocateSpec = *createRelocateSpec() + return types.BaseClusterAction(&action) +} + +func createInvalidRecommendation() types.ClusterRecommendation { + r := types.ClusterRecommendation{} + r.Reason = string(types.RecommendationReasonCodeXvmotionPlacement) + r.Action = append(r.Action, createStoragePlacementAction()) + r.Action = append(r.Action, createInvalidPlacementAction()) + return r +} + +func createValidRecommendation() (types.ClusterRecommendation, *types.VirtualMachineRelocateSpec) { + r := createInvalidRecommendation() + a, s := createValidPlacementAction() + r.Action = append(r.Action, a) + return r, s +} + +var _ = Describe("ParsePlaceVMResponse", func() { + + Context("when response is valid", func() { + Specify("PlaceVm Response is valid", func() { + res := types.PlacementResult{} + res.Recommendations = append(res.Recommendations, createInvalidRecommendation(), createInvalidRecommendation()) + rec, _ := createValidRecommendation() + rec.Reason = string(types.RecommendationReasonCodePowerOnVm) + res.Recommendations = append(res.Recommendations, rec) + rec, spec := createValidRecommendation() + res.Recommendations = append(res.Recommendations, rec) + + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).NotTo(BeNil()) + Expect(rSpec.Host).To(BeEquivalentTo(spec.Host)) + Expect(rSpec.Pool).To(BeEquivalentTo(spec.Pool)) + Expect(rSpec.Datastore).To(BeEquivalentTo(spec.Datastore)) + }) + }) + + Context("when response is not valid", func() { + Specify("PlaceVm Response without recommendations", func() { + res := types.PlacementResult{} + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).To(BeNil()) + }) + }) + + Context("when response is not valid", func() { + Specify("PlaceVm Response with invalid recommendations only", func() { + res := types.PlacementResult{} + res.Recommendations = append(res.Recommendations, createInvalidRecommendation(), createInvalidRecommendation()) + rec, _ := createValidRecommendation() + rec.Reason = string(types.RecommendationReasonCodePowerOnVm) + res.Recommendations = append(res.Recommendations, rec) + + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).To(BeNil()) + }) + }) +}) + +var _ = Describe("CheckPlacementRelocateSpec", func() { + + Context("when relocation spec is valid", func() { + Specify("Relocation spec is valid", func() { + spec := createRelocateSpec() + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeTrue()) + }) + }) + + Context("when relocation spec is not valid", func() { + Specify("Relocation spec is nil", func() { + isValid := placement.CheckPlacementRelocateSpec(nil) + Expect(isValid).To(BeFalse()) + }) + + Specify("Host is nil", func() { + spec := createRelocateSpec() + spec.Host = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + + Specify("Pool is nil", func() { + spec := createRelocateSpec() + spec.Pool = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + + Specify("Datastore is nil", func() { + spec := createRelocateSpec() + spec.Datastore = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go b/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go new file mode 100644 index 000000000..377f4133c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("Placement", vcSimPlacement) +} + +var suite = builder.NewTestSuite() + +func TestPlacement(t *testing.T) { + suite.Register(t, "VMProvider Placement", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go b/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go new file mode 100644 index 000000000..09717487e --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go @@ -0,0 +1,333 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + goctx "context" + "fmt" + "math/rand" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +type Result struct { + ZonePlacement bool + InstanceStoragePlacement bool + ZoneName string + HostMoRef *types.ManagedObjectReference + PoolMoRef types.ManagedObjectReference + // TODO: Datastore, whatever else as we need it. +} + +func doesVMNeedPlacement(vmCtx context.VirtualMachineContextA2) (res Result, needZonePlacement, needInstanceStoragePlacement bool) { + if lib.IsWcpFaultDomainsFSSEnabled() { + res.ZonePlacement = true + + if zoneName := vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey]; zoneName != "" { + // Zone has already been selected. + res.ZoneName = zoneName + } else { + // VM does not have a zone already assigned so we need to select one. + needZonePlacement = true + } + } + + if lib.IsInstanceStorageFSSEnabled() { + if instancestorage.IsPresent(vmCtx.VM) { + res.InstanceStoragePlacement = true + + if hostMoID := vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey]; hostMoID != "" { + // Host has already been selected. + res.HostMoRef = &types.ManagedObjectReference{Type: "HostSystem", Value: hostMoID} + } else { + // VM has InstanceStorage volumes so we need to select a host. + needInstanceStoragePlacement = true + } + } + } + + return +} + +// lookupChildRPs lookups the child ResourcePool under each parent ResourcePool. A VM with a ResourcePolicy +// may specify a child ResourcePool that the VM will be created under. +func lookupChildRPs( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + rpMoIDs []string, + zoneName, childRPName string) []string { + + childRPMoIDs := make([]string, 0, len(rpMoIDs)) + + for _, rpMoID := range rpMoIDs { + rp := object.NewResourcePool(vcClient, types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + childRP, err := vcenter.GetChildResourcePool(vmCtx, rp, childRPName) + if err != nil { + vmCtx.Logger.Error(err, "Skipping this resource pool since failed to get child ResourcePool", + "zone", zoneName, "parentRPMoID", rpMoID, "childRPName", childRPName) + continue + } + + childRPMoIDs = append(childRPMoIDs, childRP.Reference().Value) + } + + return childRPMoIDs +} + +// getPlacementCandidates determines the candidate resource pools for VM placement. +func getPlacementCandidates( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + vcClient *vim25.Client, + zonePlacement bool, + childRPName string) (map[string][]string, error) { + + var zones []topologyv1.AvailabilityZone + + if zonePlacement { + z, err := topology.GetAvailabilityZones(vmCtx, client) + if err != nil { + return nil, err + } + + zones = z + } else { + // Consider candidates only within the already assigned zone. + // NOTE: GetAvailabilityZone() will return a "default" AZ when the FSS is not enabled. + zone, err := topology.GetAvailabilityZone(vmCtx, client, vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey]) + if err != nil { + return nil, err + } + + zones = append(zones, zone) + } + + candidates := map[string][]string{} + + for _, zone := range zones { + nsInfo, ok := zone.Spec.Namespaces[vmCtx.VM.Namespace] + if !ok { + continue + } + + var rpMoIDs []string + if len(nsInfo.PoolMoIDs) != 0 { + rpMoIDs = nsInfo.PoolMoIDs + } else { + rpMoIDs = []string{nsInfo.PoolMoId} + } + + if childRPName != "" { + childRPMoIDs := lookupChildRPs(vmCtx, vcClient, rpMoIDs, zone.Name, childRPName) + if len(childRPMoIDs) == 0 { + vmCtx.Logger.Info("Zone had no candidates after looking up children ResourcePools", + "zone", zone.Name, "rpMoIDs", rpMoIDs, "childRPName", childRPName) + continue + } + rpMoIDs = childRPMoIDs + } + + candidates[zone.Name] = rpMoIDs + } + + return candidates, nil +} + +func rpMoIDToCluster( + ctx goctx.Context, + vcClient *vim25.Client, + rpMoRef types.ManagedObjectReference) (*object.ClusterComputeResource, error) { + + cluster, err := object.NewResourcePool(vcClient, rpMoRef).Owner(ctx) + if err != nil { + return nil, err + } + + return object.NewClusterComputeResource(vcClient, cluster.Reference()), nil +} + +// getPlacementRecommendations calls DRS PlaceVM to determine clusters suitable for placement. +func getPlacementRecommendations( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + candidates map[string][]string, + configSpec *types.VirtualMachineConfigSpec) map[string][]Recommendation { + + recommendations := map[string][]Recommendation{} + + for zoneName, rpMoIDs := range candidates { + for _, rpMoID := range rpMoIDs { + rpMoRef := types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID} + + cluster, err := rpMoIDToCluster(vmCtx, vcClient, rpMoRef) + if err != nil { + vmCtx.Logger.Error(err, "failed to get CCR from RP", "zone", zoneName, "rpMoID", rpMoID) + continue + } + + recs, err := PlaceVMForCreate(vmCtx, cluster, configSpec) + if err != nil { + vmCtx.Logger.Error(err, "PlaceVM failed", "zone", zoneName, + "clusterMoID", cluster.Reference().Value, "rpMoID", rpMoID) + continue + } + + if len(recs) == 0 { + vmCtx.Logger.Info("No placement recommendations", "zone", zoneName, + "clusterMoID", cluster.Reference().Value, "rpMoID", rpMoID) + continue + } + + // Replace the resource pool returned by PlaceVM - that is the cluster's root RP - with + // our more specific namespace or namespace child RP since this VM needs to be under the + // more specific RP. This makes the recommendations returned here the same as what zonal + // would return. + for idx := range recs { + recs[idx].PoolMoRef = rpMoRef + } + + recommendations[zoneName] = append(recommendations[zoneName], recs...) + } + } + + vmCtx.Logger.V(5).Info("Placement recommendations", "recommendations", recommendations) + + return recommendations +} + +// getZonalPlacementRecommendations calls DRS PlaceVmsXCluster to determine clusters suitable for placement. +func getZonalPlacementRecommendations( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + candidates map[string][]string, + configSpec *types.VirtualMachineConfigSpec, + needsHost bool) map[string][]Recommendation { + + rpMOToZone := map[types.ManagedObjectReference]string{} + var candidateRPMoRefs []types.ManagedObjectReference + + for zoneName, rpMoIDs := range candidates { + for _, rpMoID := range rpMoIDs { + rpMoRef := types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID} + candidateRPMoRefs = append(candidateRPMoRefs, rpMoRef) + rpMOToZone[rpMoRef] = zoneName + } + } + + var recs []Recommendation + + if len(candidateRPMoRefs) == 1 { + // If there is only one candidate, we might be able to skip some work. + + if needsHost { + // This is a hack until PlaceVmsXCluster() supports instance storage disks. + vmCtx.Logger.Info("Falling back into non-zonal placement since the only candidate needs host selected", + "rpMoID", candidateRPMoRefs[0].Value) + return getPlacementRecommendations(vmCtx, vcClient, candidates, configSpec) + } + + recs = append(recs, Recommendation{ + PoolMoRef: candidateRPMoRefs[0], + }) + vmCtx.Logger.V(5).Info("Implied placement since there was only one candidate", "rec", recs[0]) + + } else { + var err error + + recs, err = ClusterPlaceVMForCreate(vmCtx, vcClient, candidateRPMoRefs, configSpec, needsHost) + if err != nil { + vmCtx.Logger.Error(err, "PlaceVmsXCluster failed") + return nil + } + } + + recommendations := map[string][]Recommendation{} + for _, rec := range recs { + if rpZoneName, ok := rpMOToZone[rec.PoolMoRef]; ok { + recommendations[rpZoneName] = append(recommendations[rpZoneName], rec) + } else { + vmCtx.Logger.V(4).Info("Received unexpected ResourcePool recommendation", + "poolMoRef", rec.PoolMoRef) + } + } + + vmCtx.Logger.V(5).Info("Placement recommendations", "recommendations", recommendations) + + return recommendations +} + +// MakePlacementDecision selects one of the recommendations for placement. +func MakePlacementDecision(recommendations map[string][]Recommendation) (string, Recommendation) { + // Use an explicit rand.Intn() instead of first entry returned by map iterator. + zoneNames := make([]string, 0, len(recommendations)) + for zoneName := range recommendations { + zoneNames = append(zoneNames, zoneName) + } + zoneName := zoneNames[rand.Intn(len(zoneNames))] //nolint:gosec + + recs := recommendations[zoneName] + return zoneName, recs[rand.Intn(len(recs))] //nolint:gosec +} + +// Placement determines if the VM needs placement, and if so, determines where to place the VM +// and updates the Labels and Annotations with the placement decision. +func Placement( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + vcClient *vim25.Client, + configSpec *types.VirtualMachineConfigSpec, + childRPName string) (*Result, error) { + + existingRes, zonePlacement, instanceStoragePlacement := doesVMNeedPlacement(vmCtx) + if !zonePlacement && !instanceStoragePlacement { + return &existingRes, nil + } + + candidates, err := getPlacementCandidates(vmCtx, client, vcClient, zonePlacement, childRPName) + if err != nil { + return nil, err + } + + if len(candidates) == 0 { + return nil, fmt.Errorf("no placement candidates available") + } + + // TBD: May want to get the host for vGPU and other passthru devices too. + needsHost := instanceStoragePlacement + + var recommendations map[string][]Recommendation + if zonePlacement { + recommendations = getZonalPlacementRecommendations(vmCtx, vcClient, candidates, configSpec, needsHost) + } else /* instanceStoragePlacement */ { + recommendations = getPlacementRecommendations(vmCtx, vcClient, candidates, configSpec) + } + if len(recommendations) == 0 { + return nil, fmt.Errorf("no placement recommendations available") + } + + zoneName, rec := MakePlacementDecision(recommendations) + vmCtx.Logger.V(5).Info("Placement decision result", "zone", zoneName, "recommendation", rec) + + result := &Result{ + ZonePlacement: zonePlacement, + InstanceStoragePlacement: instanceStoragePlacement, + ZoneName: zoneName, + PoolMoRef: rec.PoolMoRef, + HostMoRef: rec.HostMoRef, + } + + return result, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go b/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go new file mode 100644 index 000000000..674b1ae86 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go @@ -0,0 +1,284 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/placement" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("MakePlacementDecision", func() { + + Context("only one placement decision is possible", func() { + It("makes expected decision", func() { + recommendations := map[string][]placement.Recommendation{ + "zone1": { + placement.Recommendation{ + PoolMoRef: types.ManagedObjectReference{Type: "a", Value: "abc"}, + HostMoRef: &types.ManagedObjectReference{Type: "b", Value: "xyz"}, + }, + }, + } + + zoneName, rec := placement.MakePlacementDecision(recommendations) + Expect(zoneName).To(Equal("zone1")) + Expect(rec).To(BeElementOf(recommendations[zoneName])) + }) + }) + + Context("multiple placement candidates exist", func() { + It("makes an decision", func() { + zones := map[string][]string{ + "zone1": {"z1-host1", "z1-host2", "z1-host3"}, + "zone2": {"z2-host1", "z2-host2"}, + } + + recommendations := map[string][]placement.Recommendation{} + for zoneName, hosts := range zones { + for _, host := range hosts { + recommendations[zoneName] = append(recommendations[zoneName], + placement.Recommendation{ + PoolMoRef: types.ManagedObjectReference{Type: "a", Value: "abc"}, + HostMoRef: &types.ManagedObjectReference{Type: "b", Value: host}, + }) + } + } + + zoneName, rec := placement.MakePlacementDecision(recommendations) + Expect(zones).To(HaveKey(zoneName)) + Expect(rec).To(BeElementOf(recommendations[zoneName])) + }) + }) +}) + +func vcSimPlacement() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + testConfig builder.VCSimTestConfig + + vm *vmopv1.VirtualMachine + vmCtx context.VirtualMachineContext + configSpec *types.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + + vm = builder.DummyVirtualMachine() + vm.Name = "placement-test" + + // Other than the name ConfigSpec contents don't matter for vcsim. + configSpec = &types.VirtualMachineConfigSpec{ + Name: vm.Name, + } + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + nsInfo = ctx.CreateWorkloadNamespace() + + vm.Namespace = nsInfo.Namespace + + vmCtx = context.VirtualMachineContext{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("Zone placement", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + Context("zone already assigned", func() { + zoneName := "in the zone" + + JustBeforeEach(func() { + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("returns success with same zone", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + Expect(result).ToNot(BeNil()) + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(Equal(zoneName)) + Expect(result.HostMoRef).To(BeNil()) + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + + // Current contract is the caller must look this up based on the pre-assigned zone but + // we might want to change that later. + Expect(result.PoolMoRef.Value).To(BeEmpty()) + }) + }) + + Context("no placement candidates", func() { + JustBeforeEach(func() { + vm.Namespace = "does-not-exist" + }) + + It("returns an error", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).To(MatchError("no placement candidates available")) + Expect(result).To(BeNil()) + }) + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + Expect(result.PoolMoRef.Value).ToNot(BeEmpty()) + Expect(result.HostMoRef).To(BeNil()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + + Context("Only one zone exists", func() { + BeforeEach(func() { + testConfig.NumFaultDomains = 1 + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + Expect(result.PoolMoRef.Value).ToNot(BeEmpty()) + Expect(result.HostMoRef).To(BeNil()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + }) + + Context("VM is in child RP via ResourcePolicy", func() { + It("returns success", func() { + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + vmCtx.VM.Spec.ResourcePolicyName = resourcePolicy.Name + + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, childRPName) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + + childRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, childRPName) + Expect(childRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(childRP.Reference().Value)) + }) + }) + }) + + Context("Instance Storage Placement", func() { + + BeforeEach(func() { + testConfig.WithInstanceStorage = true + builder.AddDummyInstanceStorageVolume(vm) + }) + + When("host already assigned", func() { + const hostMoID = "foobar-host-42" + + BeforeEach(func() { + vm.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey] = hostMoID + }) + + It("returns success with same host", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).To(Equal(hostMoID)) + }) + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + }) + + When("FaultDomains FSS is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + testConfig.NumFaultDomains = 1 // Only support for non-HA "HA" + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).ToNot(BeEmpty()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + + Context("VM is in child RP via ResourcePolicy", func() { + It("returns success", func() { + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + vmCtx.VM.Spec.ResourcePolicyName = resourcePolicy.Name + + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, childRPName) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + + childRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, childRPName) + Expect(childRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(childRP.Reference().Value)) + }) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/resources/vm.go b/pkg/vmprovider/providers/vsphere2/resources/vm.go new file mode 100644 index 000000000..ddc90e713 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/resources/vm.go @@ -0,0 +1,230 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" +) + +type VirtualMachine struct { + Name string + vcVirtualMachine *object.VirtualMachine + logger logr.Logger +} + +var log = logf.Log.WithName("vmresource") + +func NewVMFromObject(objVM *object.VirtualMachine) *VirtualMachine { + return &VirtualMachine{ + Name: objVM.Name(), + vcVirtualMachine: objVM, + logger: log.WithValues("name", objVM.Name()), + } +} + +func (vm *VirtualMachine) VcVM() *object.VirtualMachine { + return vm.vcVirtualMachine +} + +func (vm *VirtualMachine) Create(ctx context.Context, folder *object.Folder, pool *object.ResourcePool, vmSpec *types.VirtualMachineConfigSpec) error { + vm.logger.V(5).Info("Create VM") + + if vm.vcVirtualMachine != nil { + return fmt.Errorf("failed to create VM %q because the VM object is already set", vm.Name) + } + + createTask, err := folder.CreateVM(ctx, *vmSpec, pool, nil) + if err != nil { + return err + } + + result, err := createTask.WaitForResult(ctx, nil) + if err != nil { + return errors.Wrapf(err, "create VM %q task failed", vm.Name) + } + + vm.vcVirtualMachine = object.NewVirtualMachine(folder.Client(), result.Result.(types.ManagedObjectReference)) + return nil +} + +func (vm *VirtualMachine) Clone(ctx context.Context, folder *object.Folder, cloneSpec *types.VirtualMachineCloneSpec) (*types.ManagedObjectReference, error) { + vm.logger.V(5).Info("Clone VM") + + cloneTask, err := vm.vcVirtualMachine.Clone(ctx, folder, cloneSpec.Config.Name, *cloneSpec) + if err != nil { + return nil, err + } + + result, err := cloneTask.WaitForResult(ctx, nil) + if err != nil { + return nil, errors.Wrapf(err, "clone VM task failed") + } + + ref := result.Result.(types.ManagedObjectReference) + return &ref, nil +} + +func (vm *VirtualMachine) Reconfigure(ctx context.Context, configSpec *types.VirtualMachineConfigSpec) error { + vm.logger.V(5).Info("Reconfiguring VM", "configSpec", configSpec) + + reconfigureTask, err := vm.vcVirtualMachine.Reconfigure(ctx, *configSpec) + if err != nil { + return err + } + + _, err = reconfigureTask.WaitForResult(ctx, nil) + if err != nil { + return errors.Wrapf(err, "reconfigure VM task failed") + } + + return nil +} + +func (vm *VirtualMachine) GetProperties(ctx context.Context, properties []string) (*mo.VirtualMachine, error) { + var o mo.VirtualMachine + err := vm.vcVirtualMachine.Properties(ctx, vm.vcVirtualMachine.Reference(), properties, &o) + if err != nil { + vm.logger.Error(err, "Error getting VM properties") + return nil, err + } + + return &o, nil +} + +func (vm *VirtualMachine) ReferenceValue() string { + vm.logger.V(5).Info("Get ReferenceValue") + return vm.vcVirtualMachine.Reference().Value +} + +func (vm *VirtualMachine) MoRef() types.ManagedObjectReference { + vm.logger.V(5).Info("Get MoRef") + return vm.vcVirtualMachine.Reference() +} + +func (vm *VirtualMachine) UniqueID(ctx context.Context) (string, error) { + // Notes from Alkesh Shah regarding MoIDs in VC as of 7.0 + // + // MoRef IDs are unique within the scope of a single VC. Since Clusters are entities in VCs, the MoRef IDs will be unique across clusters + // + // Identity in VC is derived from a sequence. This ID is used in generating the MoId (or MoRef ID) for the entity in VC. Sequence is monotonically + // increasing and so during regular operation there are no dupes + // + // Backup-Restore: We now make sure that our sequence does not go back in time when restoring from a backup + // ( ) So this + // ensures that after restore we get new MoIds which are never used before… (we advance the sequence counter based on time) + // + // Discovery of VMs: We only use moids from the VM store during restore from a backup. In the unlikely event + // that there are two VMs which are presenting the same MoId, we will regenerate a new MoId based on the current + // sequence. Keep in mind, Ideally the unlikely scenario should not occur as we attempt to tamper proof the MoId + // stored in the VM store ( ) + // so two VMs having the same MoId should not happen because they cannot have the same VMX path and we use VMX path + // for ensuring this tamper proof behavior. + // + // Removing from VC and Re-adding the VM to same VC: VM will be given a new MoId (even if the VM is added using + // RegisterVM operation from VC) + // Basically, lifetime of the identity is tied to VC’s knowledge of it’s existence in it’s inventory + return vm.ReferenceValue(), nil +} + +func (vm *VirtualMachine) SetPowerState( + ctx context.Context, + currentPowerState, + desiredPowerState vmopv1.VirtualMachinePowerState, + desiredPowerOpMode vmopv1.VirtualMachinePowerOpMode) error { + + _, err := vmutil.SetAndWaitOnPowerState( + ctx, + vm.VcVM().Client(), + mo.VirtualMachine{ + ManagedEntity: mo.ManagedEntity{ + ExtensibleManagedObject: mo.ExtensibleManagedObject{ + Self: vm.VcVM().Reference(), + }, + }, + Summary: types.VirtualMachineSummary{ + Runtime: types.VirtualMachineRuntimeInfo{ + PowerState: vmutil.ParsePowerState(string(currentPowerState)), + }, + }, + }, + false, + vmutil.ParsePowerState(string(desiredPowerState)), + vmutil.ParsePowerOpMode(string(desiredPowerOpMode))) + + return err +} + +// GetVirtualDevices returns the VMs VirtualDeviceList. +func (vm *VirtualMachine) GetVirtualDevices(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(5).Info("GetVirtualDevices") + deviceList, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return deviceList, err +} + +// GetVirtualDisks returns the list of VMs vmdks. +func (vm *VirtualMachine) GetVirtualDisks(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(5).Info("GetVirtualDisks") + deviceList, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return deviceList.SelectByType((*types.VirtualDisk)(nil)), nil +} + +func (vm *VirtualMachine) GetNetworkDevices(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(4).Info("GetNetworkDevices") + devices, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return devices.SelectByType((*types.VirtualEthernetCard)(nil)), nil +} + +func (vm *VirtualMachine) Customize(ctx context.Context, spec types.CustomizationSpec) error { + vm.logger.V(5).Info("Customize", "spec", spec) + + customizeTask, err := vm.vcVirtualMachine.Customize(ctx, spec) + if err != nil { + vm.logger.Error(err, "Failed to customize VM") + return err + } + + taskInfo, err := customizeTask.WaitForResult(ctx, nil) + if err != nil { + vm.logger.Error(err, "Failed to wait for the result of Customize VM") + return err + } + + if taskErr := taskInfo.Error; taskErr != nil { + // Fetch fault messages for task.Error + fault := taskErr.Fault.GetMethodFault() + if fault != nil { + err = errors.Wrapf(err, "Fault messages: %v", fault.FaultMessage) + } + + return errors.Wrap(err, "Customization task failed") + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session.go b/pkg/vmprovider/providers/vsphere2/session/session.go new file mode 100644 index 000000000..ea0eb9881 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session.go @@ -0,0 +1,41 @@ +// Copyright (c) 2018-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" +) + +type Session struct { + Client *client.Client + K8sClient ctrlruntime.Client + Finder *find.Finder + + // Fields only used during Update + Cluster *object.ClusterComputeResource +} + +func (s *Session) invokeFsrVirtualMachine(vmCtx context.VirtualMachineContextA2, resVM *res.VirtualMachine) error { + vmCtx.Logger.Info("Invoking FSR on VM") + + task, err := internal.VirtualMachineFSR(vmCtx, resVM.MoRef(), s.Client.VimClient()) + if err != nil { + vmCtx.Logger.Error(err, "InvokeFSR call failed") + return err + } + + if err = task.Wait(vmCtx); err != nil { + vmCtx.Logger.Error(err, "InvokeFSR task failed") + return err + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go b/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go new file mode 100644 index 000000000..011bbe5b1 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestSession(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Session Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util.go b/pkg/vmprovider/providers/vsphere2/session/session_util.go new file mode 100644 index 000000000..fb25dbe91 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_util.go @@ -0,0 +1,34 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + vimTypes "github.com/vmware/govmomi/vim25/types" +) + +func ExtraConfigToMap(input []vimTypes.BaseOptionValue) (output map[string]string) { + output = make(map[string]string) + for _, opt := range input { + if optValue := opt.GetOptionValue(); optValue != nil { + // Only set string type values + if val, ok := optValue.Value.(string); ok { + output[optValue.Key] = val + } + } + } + return +} + +// MergeExtraConfig adds the key/value to the ExtraConfig if the key is not present, to let to the value be +// changed by the VM. The existing usage of ExtraConfig is hard to fit in the reconciliation model. +func MergeExtraConfig(extraConfig []vimTypes.BaseOptionValue, newMap map[string]string) []vimTypes.BaseOptionValue { + merged := make([]vimTypes.BaseOptionValue, 0) + ecMap := ExtraConfigToMap(extraConfig) + for k, v := range newMap { + if _, exists := ecMap[k]; !exists { + merged = append(merged, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + return merged +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util_test.go b/pkg/vmprovider/providers/vsphere2/session/session_util_test.go new file mode 100644 index 000000000..76ca2a0fb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_util_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2019-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" +) + +var _ = Describe("Test Session Utils", func() { + + Context("ExtraConfigToMap", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + extraConfigMap map[string]string + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{} + }) + JustBeforeEach(func() { + extraConfigMap = session.ExtraConfigToMap(extraConfig) + }) + + Context("Empty extraConfig", func() { + It("Return empty map", func() { + Expect(extraConfigMap).To(HaveLen(0)) + }) + }) + + Context("With extraConfig", func() { + BeforeEach(func() { + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key1", Value: "value1"}) + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key2", Value: "value2"}) + }) + It("Return valid map", func() { + Expect(extraConfigMap).To(HaveLen(2)) + Expect(extraConfigMap["key1"]).To(Equal("value1")) + Expect(extraConfigMap["key2"]).To(Equal("value2")) + }) + }) + }) + + Context("MergeExtraConfig", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + newMap map[string]string + merged []vimTypes.BaseOptionValue + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{Key: "existingkey1", Value: "existingvalue1"}, + &vimTypes.OptionValue{Key: "existingkey2", Value: "existingvalue2"}, + } + newMap = map[string]string{} + }) + JustBeforeEach(func() { + merged = session.MergeExtraConfig(extraConfig, newMap) + }) + + Context("Empty newMap", func() { + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with existing key", func() { + BeforeEach(func() { + newMap["existingkey1"] = "existingkey1" + }) + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with new keys", func() { + BeforeEach(func() { + newMap["newkey1"] = "newvalue1" + newMap["newkey2"] = "newvalue2" + }) + It("Return merged map", func() { + Expect(merged).To(HaveLen(2)) + mergedMap := session.ExtraConfigToMap(merged) + Expect(mergedMap["newkey1"]).To(Equal("newvalue1")) + Expect(mergedMap["newkey2"]).To(Equal("newvalue2")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm.go b/pkg/vmprovider/providers/vsphere2/session/session_vm.go new file mode 100644 index 000000000..a7aaff105 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "fmt" + + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +func updateVirtualDiskDeviceChanges( + vmCtx context.VirtualMachineContextA2, + virtualDisks object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil, nil + } + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + found := false + for _, vmDevice := range virtualDisks { + vmDisk, ok := vmDevice.(*vimTypes.VirtualDisk) + if !ok { + continue + } + + // Assume the first disk as the boot disk. We can make this smarter by + // looking at the disk path or whatever else later. + // TODO: De-dupe this with resizeBootDiskDeviceChange() in the clone path. + + newCapacityInBytes := capacity.Value() + if newCapacityInBytes < vmDisk.CapacityInBytes { + err := fmt.Errorf("cannot shrink boot disk from %d bytes to %d bytes", + vmDisk.CapacityInBytes, newCapacityInBytes) + return nil, err + } + + if vmDisk.CapacityInBytes < newCapacityInBytes { + vmDisk.CapacityInBytes = newCapacityInBytes + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationEdit, + Device: vmDisk, + }) + } + + found = true + break + } + + if !found { + return nil, fmt.Errorf("could not find the boot disk to change capacity") + } + + return deviceChanges, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go new file mode 100644 index 000000000..1154c20fa --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -0,0 +1,957 @@ +// Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "fmt" + "reflect" + "time" + + "k8s.io/utils/pointer" + + "github.com/go-logr/logr" + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + apiEquality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/util" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + network2 "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +const ( + FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" +) + +// VMUpdateArgs contains the arguments needed to update a VM on VC. +type VMUpdateArgs struct { + VMClass *vmopv1.VirtualMachineClass + ResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + MinCPUFreq uint64 + ExtraConfig map[string]string + + BootstrapData vmlifecycle.BootstrapData + + ConfigSpec *vimTypes.VirtualMachineConfigSpec + + NetworkResults network2.NetworkInterfaceResults + + // hack. Remove after VMSVC-1261. + // indicating if this VM image used is VM service v1alpha1 compatible. + VirtualMachineImageV1Alpha1Compatible bool +} + +func ethCardMatch(newBaseEthCard, curBaseEthCard vimTypes.BaseVirtualEthernetCard) bool { + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if reflect.TypeOf(curBaseEthCard) != reflect.TypeOf(newBaseEthCard) { + return false + } + } + + curEthCard := curBaseEthCard.GetVirtualEthernetCard() + newEthCard := newBaseEthCard.GetVirtualEthernetCard() + if newEthCard.AddressType == string(vimTypes.VirtualEthernetCardMacTypeManual) { + // If the new card has an assigned MAC address, then it should match with + // the current card. Note only NCP sets the MAC address. + if newEthCard.MacAddress != curEthCard.MacAddress { + return false + } + } + + if newEthCard.ExternalId != "" { + // If the new card has a specific ExternalId, then it should match with the + // current card. Note only NCP sets the ExternalId. + if newEthCard.ExternalId != curEthCard.ExternalId { + return false + } + } + + return true +} + +func UpdateEthCardDeviceChanges( + expectedEthCards object.VirtualDeviceList, + currentEthCards object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + for _, expectedDev := range expectedEthCards { + expectedNic := expectedDev.(vimTypes.BaseVirtualEthernetCard) + expectedBacking := expectedNic.GetVirtualEthernetCard().Backing + expectedBackingType := reflect.TypeOf(expectedBacking) + + var matchingIdx = -1 + + // Try to match the expected NIC with an existing NIC but this isn't that great. We mostly + // depend on the backing but we can improve that later on. When not generated, we could use + // the MAC address. When we support something other than just vmxnet3 we should compare + // those types too. And we should make this truly reconcile as well by comparing the full + // state (support EDIT instead of only ADD/REMOVE operations). + // + // Another tack we could take is force the VM's device order to match the Spec order, but + // that could lead to spurious removals. Or reorder the NetIfList to not be that of the + // Spec, but in VM device order. + for idx, curDev := range currentEthCards { + nic := curDev.(vimTypes.BaseVirtualEthernetCard) + + // This assumes we don't have multiple NICs in the same backing network. This is kind of, sort + // of enforced by the webhook, but we lack a guaranteed way to match up the NICs. + + if !ethCardMatch(expectedNic, nic) { + continue + } + + db := nic.GetVirtualEthernetCard().Backing + if db == nil || reflect.TypeOf(db) != expectedBackingType { + continue + } + + var backingMatch bool + + // Cribbed from VirtualDeviceList.SelectByBackingInfo(). + switch a := db.(type) { + case *vimTypes.VirtualEthernetCardNetworkBackingInfo: + // This backing is only used in testing. + b := expectedBacking.(*vimTypes.VirtualEthernetCardNetworkBackingInfo) + backingMatch = a.DeviceName == b.DeviceName + case *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo: + b := expectedBacking.(*vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo) + backingMatch = a.Port.SwitchUuid == b.Port.SwitchUuid && a.Port.PortgroupKey == b.Port.PortgroupKey + case *vimTypes.VirtualEthernetCardOpaqueNetworkBackingInfo: + b := expectedBacking.(*vimTypes.VirtualEthernetCardOpaqueNetworkBackingInfo) + backingMatch = a.OpaqueNetworkId == b.OpaqueNetworkId + } + + if backingMatch { + matchingIdx = idx + break + } + } + + if matchingIdx == -1 { + // No matching backing found so add new card. + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: expectedDev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + }) + } else { + // Matching backing found so keep this card (don't remove it below after this loop). + currentEthCards = append(currentEthCards[:matchingIdx], currentEthCards[matchingIdx+1:]...) + } + } + + // Remove any unmatched existing interfaces. + removeDeviceChanges := make([]vimTypes.BaseVirtualDeviceConfigSpec, 0, len(currentEthCards)) + for _, dev := range currentEthCards { + removeDeviceChanges = append(removeDeviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: dev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationRemove, + }) + } + + // Process any removes first. + return append(removeDeviceChanges, deviceChanges...), nil +} + +// UpdatePCIDeviceChanges returns devices changes for PCI devices attached to a VM. There are 2 types of PCI devices +// processed here and in case of cloning a VM, devices listed in VMClass are considered as source of truth. +func UpdatePCIDeviceChanges( + expectedPciDevices object.VirtualDeviceList, + currentPciDevices object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + for _, expectedDev := range expectedPciDevices { + expectedPci := expectedDev.(*vimTypes.VirtualPCIPassthrough) + expectedBacking := expectedPci.Backing + expectedBackingType := reflect.TypeOf(expectedBacking) + + var matchingIdx = -1 + for idx, curDev := range currentPciDevices { + curBacking := curDev.GetVirtualDevice().Backing + if curBacking == nil || reflect.TypeOf(curBacking) != expectedBackingType { + continue + } + + var backingMatch bool + switch a := curBacking.(type) { + case *vimTypes.VirtualPCIPassthroughVmiopBackingInfo: + b := expectedBacking.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo) + backingMatch = a.Vgpu == b.Vgpu + + case *vimTypes.VirtualPCIPassthroughDynamicBackingInfo: + currAllowedDevs := a.AllowedDevice + b := expectedBacking.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo) + if a.CustomLabel == b.CustomLabel { + // b.AllowedDevice has only one element because CreatePCIDevices() adds only one device based + // on the devices listed in vmclass.spec.hardware.devices.dynamicDirectPathIODevices. + expectedAllowedDev := b.AllowedDevice[0] + for i := 0; i < len(currAllowedDevs) && !backingMatch; i++ { + backingMatch = expectedAllowedDev.DeviceId == currAllowedDevs[i].DeviceId && + expectedAllowedDev.VendorId == currAllowedDevs[i].VendorId + } + } + } + + if backingMatch { + matchingIdx = idx + break + } + } + + if matchingIdx == -1 { + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: expectedPci, + }) + } else { + // There could be multiple vGPUs with same BackingInfo. Remove current device if matching found. + currentPciDevices = append(currentPciDevices[:matchingIdx], currentPciDevices[matchingIdx+1:]...) + } + } + // Remove any unmatched existing devices. + removeDeviceChanges := make([]vimTypes.BaseVirtualDeviceConfigSpec, 0, len(currentPciDevices)) + for _, dev := range currentPciDevices { + removeDeviceChanges = append(removeDeviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: dev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationRemove, + }) + } + + // Process any removes first. + return append(removeDeviceChanges, deviceChanges...), nil +} + +func UpdateConfigSpecCPUAllocation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + minCPUFreq uint64) { + + cpuAllocation := config.CpuAllocation + var cpuReservation *int64 + var cpuLimit *int64 + + if !vmClassSpec.Policies.Resources.Requests.Cpu.IsZero() { + rsv := virtualmachine.CPUQuantityToMhz(vmClassSpec.Policies.Resources.Requests.Cpu, minCPUFreq) + if cpuAllocation == nil || cpuAllocation.Reservation == nil || *cpuAllocation.Reservation != rsv { + cpuReservation = &rsv + } + } + + if !vmClassSpec.Policies.Resources.Limits.Cpu.IsZero() { + lim := virtualmachine.CPUQuantityToMhz(vmClassSpec.Policies.Resources.Limits.Cpu, minCPUFreq) + if cpuAllocation == nil || cpuAllocation.Limit == nil || *cpuAllocation.Limit != lim { + cpuLimit = &lim + } + } + + if cpuReservation != nil || cpuLimit != nil { + configSpec.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: cpuReservation, + Limit: cpuLimit, + } + } +} + +func UpdateConfigSpecMemoryAllocation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec) { + + memAllocation := config.MemoryAllocation + var memoryReservation *int64 + var memoryLimit *int64 + + if !vmClassSpec.Policies.Resources.Requests.Memory.IsZero() { + rsv := virtualmachine.MemoryQuantityToMb(vmClassSpec.Policies.Resources.Requests.Memory) + if memAllocation == nil || memAllocation.Reservation == nil || *memAllocation.Reservation != rsv { + memoryReservation = &rsv + } + } + + if !vmClassSpec.Policies.Resources.Limits.Memory.IsZero() { + lim := virtualmachine.MemoryQuantityToMb(vmClassSpec.Policies.Resources.Limits.Memory) + if memAllocation == nil || memAllocation.Limit == nil || *memAllocation.Limit != lim { + memoryLimit = &lim + } + } + + if memoryReservation != nil || memoryLimit != nil { + configSpec.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: memoryReservation, + Limit: memoryLimit, + } + } +} + +func UpdateConfigSpecExtraConfig( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + vm *vmopv1.VirtualMachine, + globalExtraConfig map[string]string, + imageV1Alpha1Compatible bool) { + + extraConfig := make(map[string]string) + for k, v := range globalExtraConfig { + extraConfig[k] = v + } + + virtualDevices := vmClassSpec.Hardware.Devices + pciPassthruFromConfigSpec := util.SelectVirtualPCIPassthrough(util.DevicesFromConfigSpec(classConfigSpec)) + if len(virtualDevices.VGPUDevices) > 0 || len(virtualDevices.DynamicDirectPathIODevices) > 0 || len(pciPassthruFromConfigSpec) > 0 { + // Add "maintenance.vm.evacuation.poweroff" extraConfig key when GPU devices are present in the VMClass Spec. + extraConfig[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + setMMIOExtraConfig(vm, extraConfig) + } + + // If VM has InstanceStorage configured, add "maintenance.vm.evacuation.poweroff" to extraConfig + if instancestorage.IsPresent(vm) { + extraConfig[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + // Merge non intersecting keys from the desired config spec extra config with the class config spec extra config + // (ie) class config spec extra config keys takes precedence over the desired config spec extra config keys + ecFromClassConfigSpec := ExtraConfigToMap(classConfigSpec.ExtraConfig) + mergedExtraConfig := classConfigSpec.ExtraConfig + for k, v := range extraConfig { + if _, exists := ecFromClassConfigSpec[k]; !exists { + mergedExtraConfig = append(mergedExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + extraConfig = ExtraConfigToMap(mergedExtraConfig) + } + + configSpec.ExtraConfig = MergeExtraConfig(config.ExtraConfig, extraConfig) + + // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot + // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled + // when the vmMetadata is nil or when the transport requested is not CloudInit. + // VMSVC-1261: we may always set this extra config key to remove image from VM customization. + // If a VM is deployed from an incompatible image, + // it will do nothing and won't cause any issues, but can introduce confusion. + // BMV: Is this needed anymore? IMO we shouldn't have bootstrap stuff here. The EC mangling is already hard to follow. + emptyBSSpec := vmopv1.VirtualMachineBootstrapSpec{} + if vm.Spec.Bootstrap == emptyBSSpec || vm.Spec.Bootstrap.CloudInit == nil { + ecMap := ExtraConfigToMap(config.ExtraConfig) + if ecMap[constants.VMOperatorV1Alpha1ExtraConfigKey] == constants.VMOperatorV1Alpha1ConfigReady && + imageV1Alpha1Compatible { + // Set VMOperatorV1Alpha1ExtraConfigKey for v1alpha1 VirtualMachineImage compatibility. + configSpec.ExtraConfig = append(configSpec.ExtraConfig, + &vimTypes.OptionValue{Key: constants.VMOperatorV1Alpha1ExtraConfigKey, Value: constants.VMOperatorV1Alpha1ConfigEnabled}) + } + } +} + +func setMMIOExtraConfig(vm *vmopv1.VirtualMachine, extraConfig map[string]string) { + mmioSize := vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] + if mmioSize == "" { + mmioSize = constants.PCIPassthruMMIOSizeDefault + } + if mmioSize != "0" { + extraConfig[constants.PCIPassthruMMIOExtraConfigKey] = constants.ExtraConfigTrue + extraConfig[constants.PCIPassthruMMIOSizeExtraConfigKey] = mmioSize + } +} + +func UpdateConfigSpecChangeBlockTracking( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec, + vmSpec vmopv1.VirtualMachineSpec) { + + // When VM_Class_as_Config_DaynDate is enabled, class config spec cbt if + // set overrides the VM spec advanced options cbt. + // BMV: I don't think this is correct: the class shouldn't dictate this for backup purposes. There is a + // webhook out there that changes this in the VM spec. + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && classConfigSpec != nil { + if classConfigSpec.ChangeTrackingEnabled != nil { + if !apiEquality.Semantic.DeepEqual(config.ChangeTrackingEnabled, classConfigSpec.ChangeTrackingEnabled) { + configSpec.ChangeTrackingEnabled = classConfigSpec.ChangeTrackingEnabled + } + return + } + } + + if vmSpec.Advanced.ChangeBlockTracking { + if config.ChangeTrackingEnabled == nil || !*config.ChangeTrackingEnabled { + configSpec.ChangeTrackingEnabled = pointer.Bool(true) + } + } else { + if config.ChangeTrackingEnabled != nil && *config.ChangeTrackingEnabled { + configSpec.ChangeTrackingEnabled = pointer.Bool(false) + } + } +} + +func UpdateHardwareConfigSpec( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec) { + + if nCPUs := int32(vmClassSpec.Hardware.Cpus); config.Hardware.NumCPU != nCPUs { + configSpec.NumCPUs = nCPUs + } + if memMB := virtualmachine.MemoryQuantityToMb(vmClassSpec.Hardware.Memory); int64(config.Hardware.MemoryMB) != memMB { + configSpec.MemoryMB = memMB + } +} + +func UpdateConfigSpecAnnotation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec) { + if config.Annotation != constants.VCVMAnnotation { + configSpec.Annotation = constants.VCVMAnnotation + } +} + +func UpdateConfigSpecManagedBy( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec) { + if config.ManagedBy == nil { + configSpec.ManagedBy = &vimTypes.ManagedByInfo{ + ExtensionKey: constants.ManagedByExtensionKey, + Type: constants.ManagedByExtensionType, + } + } +} + +func UpdateConfigSpecFirmware( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vm *vmopv1.VirtualMachine) { + + if val, ok := vm.Annotations[constants.FirmwareOverrideAnnotation]; ok { + if (val == "efi" || val == "bios") && config.Firmware != val { + configSpec.Firmware = val + } + } +} + +// UpdateConfigSpecDeviceGroups sets the desired config spec device groups to reconcile by differencing the +// current VM config and the class config spec device groups. +func UpdateConfigSpecDeviceGroups( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec) { + + if classConfigSpec.DeviceGroups != nil { + if config.DeviceGroups == nil || !reflect.DeepEqual(classConfigSpec.DeviceGroups.DeviceGroup, config.DeviceGroups.DeviceGroup) { + configSpec.DeviceGroups = classConfigSpec.DeviceGroups + } + } +} + +// updateConfigSpec overlays the VM Class spec with the provided ConfigSpec to form a desired +// ConfigSpec that will be used to reconfigure the VM. +func updateConfigSpec( + vmCtx context.VirtualMachineContextA2, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) *vimTypes.VirtualMachineConfigSpec { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + vmClassSpec := updateArgs.VMClass.Spec + + // Before VM Class as Config, VMs were deployed from the OVA, and are then + // reconfigured to match the desired CPU and memory reservation. Maintain that + // behavior. With the FSS enabled, VMs will be _created_ with desired HW spec, and we + // will not modify the hardware of the VM post creation. So, don't populate the + // Hardware config and CPU/Memory reservation. + if !lib.IsVMClassAsConfigFSSDaynDateEnabled() { + UpdateHardwareConfigSpec(config, configSpec, &vmClassSpec) + UpdateConfigSpecCPUAllocation(config, configSpec, &vmClassSpec, updateArgs.MinCPUFreq) + UpdateConfigSpecMemoryAllocation(config, configSpec, &vmClassSpec) + } + + UpdateConfigSpecAnnotation(config, configSpec) + UpdateConfigSpecManagedBy(config, configSpec) + UpdateConfigSpecExtraConfig(config, configSpec, updateArgs.ConfigSpec, &vmClassSpec, + vmCtx.VM, updateArgs.ExtraConfig, updateArgs.VirtualMachineImageV1Alpha1Compatible) + UpdateConfigSpecChangeBlockTracking(config, configSpec, updateArgs.ConfigSpec, vmCtx.VM.Spec) + UpdateConfigSpecFirmware(config, configSpec, vmCtx.VM) + UpdateConfigSpecDeviceGroups(config, configSpec, updateArgs.ConfigSpec) + + return configSpec +} + +func (s *Session) prePowerOnVMConfigSpec( + vmCtx context.VirtualMachineContextA2, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) (*vimTypes.VirtualMachineConfigSpec, error) { + + configSpec := updateConfigSpec(vmCtx, config, updateArgs) + + virtualDevices := object.VirtualDeviceList(config.Hardware.Device) + currentDisks := virtualDevices.SelectByType((*vimTypes.VirtualDisk)(nil)) + currentEthCards := virtualDevices.SelectByType((*vimTypes.VirtualEthernetCard)(nil)) + currentPciDevices := virtualDevices.SelectByType((*vimTypes.VirtualPCIPassthrough)(nil)) + + diskDeviceChanges, err := updateVirtualDiskDeviceChanges(vmCtx, currentDisks) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, diskDeviceChanges...) + + var expectedEthCards object.VirtualDeviceList + for idx := range updateArgs.NetworkResults.Results { + expectedEthCards = append(expectedEthCards, updateArgs.NetworkResults.Results[idx].Device) + } + + ethCardDeviceChanges, err := UpdateEthCardDeviceChanges(expectedEthCards, currentEthCards) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, ethCardDeviceChanges...) + + var expectedPCIDevices []vimTypes.BaseVirtualDevice + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if configSpecDevs := util.DevicesFromConfigSpec(updateArgs.ConfigSpec); len(configSpecDevs) > 0 { + pciPassthruFromConfigSpec := util.SelectVirtualPCIPassthrough(configSpecDevs) + expectedPCIDevices = virtualmachine.CreatePCIDevicesFromConfigSpec(pciPassthruFromConfigSpec) + } + } else { + expectedPCIDevices = virtualmachine.CreatePCIDevicesFromVMClass(updateArgs.VMClass.Spec.Hardware.Devices) + } + + pciDeviceChanges, err := UpdatePCIDeviceChanges(expectedPCIDevices, currentPciDevices) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, pciDeviceChanges...) + + return configSpec, nil +} + +func (s *Session) prePowerOnVMReconfigure( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + configSpec, err := s.prePowerOnVMConfigSpec(vmCtx, config, updateArgs) + if err != nil { + return err + } + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("Pre PowerOn Reconfigure", "configSpec", configSpec) + if err := resVM.Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "pre power on reconfigure failed") + return err + } + } + + return nil +} + +func (s *Session) ensureNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + configSpec *vimTypes.VirtualMachineConfigSpec) (network2.NetworkInterfaceResults, error) { + + // This negative device key is the traditional range used for network interfaces. + deviceKey := int32(-100) + + var networkDevices []vimTypes.BaseVirtualDevice + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && configSpec != nil { + networkDevices = util.SelectDevicesByTypes( + util.DevicesFromConfigSpec(configSpec), + &vimTypes.VirtualE1000{}, + &vimTypes.VirtualE1000e{}, + &vimTypes.VirtualPCNet32{}, + &vimTypes.VirtualVmxnet2{}, + &vimTypes.VirtualVmxnet3{}, + &vimTypes.VirtualVmxnet3Vrdma{}, + &vimTypes.VirtualSriovEthernetCard{}, + ) + } + networkSpec := &vmCtx.VM.Spec.Network + if networkSpec.Disabled { + // No connected networking for this VM. + return network2.NetworkInterfaceResults{}, nil + } + + interfaces := networkSpec.Interfaces + if len(interfaces) == 0 { + // VM gets one automatic NIC. Create the default interface from fields in the network spec. + defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: networkSpec.DeviceName, + Addresses: networkSpec.Addresses, + DHCP4: networkSpec.DHCP4, + DHCP6: networkSpec.DHCP6, + Gateway4: networkSpec.Gateway4, + Gateway6: networkSpec.Gateway6, + MTU: networkSpec.MTU, + Nameservers: networkSpec.Nameservers, + Routes: networkSpec.Routes, + SearchDomains: networkSpec.SearchDomains, + } + + if defaultInterface.Name == "" { + defaultInterface.Name = "eth0" + } + if networkSpec.Network != nil { + defaultInterface.Network = *networkSpec.Network + } + + interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} + } + + clusterMoRef := s.Cluster.Reference() + results, err := network2.CreateAndWaitForNetworkInterfaces( + vmCtx, + s.K8sClient, + s.Client.VimClient(), + s.Finder, + &clusterMoRef, + interfaces) + if err != nil { + return network2.NetworkInterfaceResults{}, err + } + + // XXX: The following logic assumes that the order of network interfaces specified in the + // VM spec matches one to one with the device changes in the ConfigSpec in VM class. + // This is a safe assumption for now since VM service only supports one network interface. + // TODO: Needs update when VM Service supports VMs with more then one network interface. + for idx := range results.Results { + result := &results.Results[idx] + + dev, err := network2.CreateDefaultEthCard(vmCtx, result) + if err != nil { + return network2.NetworkInterfaceResults{}, err + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + // If VM Class-as-a-Config is supported, we use the network device from the Class. + // If the VM class doesn't specify enough number of network devices, we fall back to default behavior. + if idx < len(networkDevices) { + ethCardFromNetProvider := dev.(vimTypes.BaseVirtualEthernetCard) + + if mac := ethCardFromNetProvider.GetVirtualEthernetCard().MacAddress; mac != "" { + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = mac + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + } + + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = + ethCardFromNetProvider.GetVirtualEthernetCard().ExternalId + // If the device from VM class has a DVX backing, this should still work if the backing as well + // as the DVX backing are set. VPXD checks for DVX backing before checking for normal device backings. + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().Backing = + ethCardFromNetProvider.GetVirtualEthernetCard().Backing + + dev = networkDevices[idx] + } + } + + // govmomi assigns a random device key. Fix that up here. + dev.GetVirtualDevice().Key = deviceKey + deviceKey-- + + result.Device = dev + } + + return results, nil +} + +func (s *Session) ensureCNSVolumes(vmCtx context.VirtualMachineContextA2) error { + // If VM spec has a PVC, check if the volume is attached before powering on + for _, volume := range vmCtx.VM.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + // Only handle PVC volumes here. In v1a1 we had non-PVC ("vsphereVolumes") but those are gone. + continue + } + + // BMV: We should not use the Status as the SoT here. What a mess. + found := false + for _, volumeStatus := range vmCtx.VM.Status.Volumes { + if volumeStatus.Name == volume.Name { + found = true + if !volumeStatus.Attached { + return fmt.Errorf("persistent volume: %s not attached to VM", volume.Name) + } + break + } + } + + if !found { + return fmt.Errorf("status update pending for persistent volume: %s on VM", volume.Name) + } + } + + return nil +} + +func (s *Session) customize( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + cfg *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + { + // Hack: the old code only populated the first nic address - getFirstNicMacAddr() - so just keep + // doing the same here. We need a generalized UpdateEthCardDeviceChanges() to match up the Spec + // with the actual devices. Old code also made this best effort so do that here too. + // I've got a larger change that removes the old session stuff, and improves on all this behavior + // but I didn't have the BW to sort out all the changes. + if len(updateArgs.NetworkResults.Results) > 1 { + mac := updateArgs.NetworkResults.Results[0].MacAddress + if mac == "" { + ethCards, _ := resVM.GetNetworkDevices(vmCtx) + if len(ethCards) > 0 { + curNic := ethCards[0].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard() + updateArgs.NetworkResults.Results[0].MacAddress = curNic.GetVirtualEthernetCard().MacAddress + } + } + } + } + + err := vmlifecycle.DoBootstrap(vmCtx, resVM.VcVM(), cfg, s.K8sClient, updateArgs.NetworkResults, updateArgs.BootstrapData) + if err != nil { + return err + } + + return nil +} + +func (s *Session) prepareVMForPowerOn( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + cfg *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + netIfList, err := s.ensureNetworkInterfaces(vmCtx, updateArgs.ConfigSpec) + if err != nil { + return err + } + + updateArgs.NetworkResults = netIfList + + err = s.prePowerOnVMReconfigure(vmCtx, resVM, cfg, updateArgs) + if err != nil { + return err + } + + err = s.customize(vmCtx, resVM, cfg, updateArgs) + if err != nil { + return err + } + + err = s.ensureCNSVolumes(vmCtx) + if err != nil { + return err + } + + return nil +} + +func (s *Session) poweredOnVMReconfigure( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo) error { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + UpdateConfigSpecChangeBlockTracking(config, configSpec, nil, vmCtx.VM.Spec) + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("PoweredOn Reconfigure", "configSpec", configSpec) + if err := resVM.Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "powered on reconfigure failed") + return err + } + + // Special case for CBT: in order for CBT change take effect for a powered on VM, + // a checkpoint save/restore is needed. tracks the implementation of + // this FSR internally to vSphere. + if configSpec.ChangeTrackingEnabled != nil { + if err := s.invokeFsrVirtualMachine(vmCtx, resVM); err != nil { + vmCtx.Logger.Error(err, "Failed to invoke FSR for CBT update") + return err + } + } + } + + return nil +} + +func (s *Session) attachClusterModule( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + // The clusterModule is required be able to enforce the vm-vm anti-affinity policy. + clusterModuleName := vmCtx.VM.Annotations[pkg.ClusterModuleNameKey] + if clusterModuleName == "" { + return nil + } + + // Find ClusterModule UUID from the ResourcePolicy. + _, moduleUUID := clustermodules.FindClusterModuleUUID(clusterModuleName, s.Cluster.Reference(), resourcePolicy) + if moduleUUID == "" { + return fmt.Errorf("ClusterModule %s not found", clusterModuleName) + } + + return s.Client.ClusterModuleClient().AddMoRefToModule(vmCtx, moduleUUID, resVM.MoRef()) +} + +func (s *Session) UpdateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + getUpdateArgsFn func() (*VMUpdateArgs, error)) (err error) { + + resVM := res.NewVMFromObject(vcVM) + + moVM, err := resVM.GetProperties(vmCtx, []string{"config", "runtime"}) + if err != nil { + return err + } + + defer func() { + updateErr := vmlifecycle.UpdateStatus(vmCtx, s.K8sClient, vcVM, nil) + if updateErr != nil { + vmCtx.Logger.Error(updateErr, "Updating VM status failed") + if err == nil { + err = updateErr + } + } + }() + + // Translate the VM's current power state into the VM Op power state value. + var existingPowerState vmopv1.VirtualMachinePowerState + switch moVM.Runtime.PowerState { + case vimTypes.VirtualMachinePowerStatePoweredOn: + existingPowerState = vmopv1.VirtualMachinePowerStateOn + case vimTypes.VirtualMachinePowerStatePoweredOff: + existingPowerState = vmopv1.VirtualMachinePowerStateOff + case vimTypes.VirtualMachinePowerStateSuspended: + existingPowerState = vmopv1.VirtualMachinePowerStateSuspended + } + + switch vmCtx.VM.Spec.PowerState { + case vmopv1.VirtualMachinePowerStateOff: + var powerOff bool + if existingPowerState == vmopv1.VirtualMachinePowerStateOn { + powerOff = true + } else if existingPowerState == vmopv1.VirtualMachinePowerStateSuspended { + if vmCtx.VM.Spec.PowerOffMode == vmopv1.VirtualMachinePowerOpModeHard { + powerOff = true + } + } + if powerOff { + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmCtx.VM.Spec.PowerOffMode) + } + + // BMV: We'll likely want to reconfigure a powered off VM too, but right now + // we'll defer that until the pre power on (and until more people complain + // that the UI appears wrong). + + case vmopv1.VirtualMachinePowerStateSuspended: + if existingPowerState == vmopv1.VirtualMachinePowerStateOn { + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmCtx.VM.Spec.SuspendMode) + } + + case vmopv1.VirtualMachinePowerStateOn: + config := moVM.Config + + // See GoVmomi's VirtualMachine::Device() explanation for this check. + if config == nil { + return fmt.Errorf("VM config is not available, connectionState=%s", moVM.Runtime.ConnectionState) + } + + switch existingPowerState { + case vmopv1.VirtualMachinePowerStateOn: + + // Check to see if a possible restart is required. + // Please note a VM may only be restarted if it is powered on. + if vmCtx.VM.Spec.NextRestartTime != "" { + + // If non-empty, the value of spec.nextRestartTime is guaranteed + // to be a valid RFC3339Nano timestamp due to the webhooks, + // however, we still check for the error due to testing that may + // not involve webhooks. + nextRestartTime, err := time.Parse( + time.RFC3339Nano, vmCtx.VM.Spec.NextRestartTime) + if err != nil { + return fmt.Errorf( + "spec.nextRestartTime %q cannot be parsed with %q %w", + vmCtx.VM.Spec.NextRestartTime, + time.RFC3339Nano, + err) + } + result, err := vmutil.RestartAndWait( + logr.NewContext(vmCtx, vmCtx.Logger), + vcVM.Client(), + vmutil.ManagedObjectFromObject(vcVM), + false, + nextRestartTime, + vmutil.ParsePowerOpMode(string(vmCtx.VM.Spec.RestartMode))) + if err != nil { + return err + } + if result.AnyChange() { + lastRestartTime := metav1.NewTime(nextRestartTime) + vmCtx.VM.Status.LastRestartTime = &lastRestartTime + } + } + + // Do not pass classConfigSpec to poweredOnVMReconfigure when VM is + // already powered on since we do not have to get VM class at this + // point. + return s.poweredOnVMReconfigure(vmCtx, resVM, config) + + case vmopv1.VirtualMachinePowerStateSuspended: + // A suspended VM cannot be reconfigured. + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmopv1.VirtualMachinePowerOpModeHard) + } + + updateArgs, err := getUpdateArgsFn() + if err != nil { + return err + } + + // TODO: Find a better place for this? + if err := s.attachClusterModule(vmCtx, resVM, updateArgs.ResourcePolicy); err != nil { + return err + } + + if err := s.prepareVMForPowerOn(vmCtx, resVM, config, updateArgs); err != nil { + return err + } + + if err := resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmopv1.VirtualMachinePowerOpModeHard); err != nil { + return err + } + + if vmCtx.VM.Annotations == nil { + vmCtx.VM.Annotations = map[string]string{} + } + vmCtx.VM.Annotations[FirstBootDoneAnnotation] = "true" + } + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go new file mode 100644 index 000000000..bd3098660 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go @@ -0,0 +1,1236 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var _ = Describe("Update ConfigSpec", func() { + + var ( + config *vimTypes.VirtualMachineConfigInfo + configSpec *vimTypes.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + config = &vimTypes.VirtualMachineConfigInfo{} + configSpec = &vimTypes.VirtualMachineConfigSpec{} + }) + + // Just a few examples for testing these things here. Need to think more about whether this + // is a good way or not. Probably better to do this via UpdateVirtualMachine when we have + // better integration tests. + + Context("Basic Hardware", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateHardwareConfigSpec(config, configSpec, vmClassSpec) + }) + + Context("Updates Hardware", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Cpus = 42 + vmClassSpec.Hardware.Memory = resource.MustParse("2000Mi") + }) + + It("config spec is not empty", func() { + Expect(configSpec.NumCPUs).To(BeNumerically("==", 42)) + Expect(configSpec.MemoryMB).To(BeNumerically("==", 2000)) + }) + }) + + Context("config already matches", func() { + BeforeEach(func() { + config.Hardware.NumCPU = 42 + vmClassSpec.Hardware.Cpus = int64(config.Hardware.NumCPU) + config.Hardware.MemoryMB = 1500 + vmClassSpec.Hardware.Memory = resource.MustParse(fmt.Sprintf("%dMi", config.Hardware.MemoryMB)) + }) + + It("config spec show no changes", func() { + Expect(configSpec.NumCPUs).To(BeZero()) + Expect(configSpec.MemoryMB).To(BeZero()) + }) + }) + }) + + Context("CPU Allocation", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + var minCPUFreq uint64 = 1 + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecCPUAllocation(config, configSpec, vmClassSpec, minCPUFreq) + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + + Context("config matches class policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Requests.Cpu = r + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + }) + + Context("config matches class policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Limits.Cpu = r + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + }) + + Context("config matches is different from policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(10 * virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Limits.Cpu = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.CpuAllocation.Reservation).To(BeNil()) + Expect(configSpec.CpuAllocation.Limit).ToNot(BeNil()) + Expect(*configSpec.CpuAllocation.Limit).To(BeNumerically("==", 100*1024*1024)) + }) + }) + + Context("config matches is different from policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(10 * virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Requests.Cpu = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.CpuAllocation.Limit).To(BeNil()) + Expect(configSpec.CpuAllocation.Reservation).ToNot(BeNil()) + Expect(*configSpec.CpuAllocation.Reservation).To(BeNumerically("==", 100*1024*1024)) + }) + }) + }) + + Context("Memory Allocation", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecMemoryAllocation(config, configSpec, vmClassSpec) + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + + Context("config matches class policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Requests.Memory = r + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + }) + + Context("config matches class policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Limits.Memory = r + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + }) + + Context("config matches is different from policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(10 * virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Limits.Memory = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation.Reservation).To(BeNil()) + Expect(configSpec.MemoryAllocation.Limit).ToNot(BeNil()) + Expect(*configSpec.MemoryAllocation.Limit).To(BeNumerically("==", 100)) + }) + }) + + Context("config matches is different from policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(10 * virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Requests.Memory = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation.Limit).To(BeNil()) + Expect(configSpec.MemoryAllocation.Reservation).ToNot(BeNil()) + Expect(*configSpec.MemoryAllocation.Reservation).To(BeNumerically("==", 100)) + }) + }) + }) + + Context("ExtraConfig", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + var vm *vmopv1.VirtualMachine + var globalExtraConfig map[string]string + var ecMap map[string]string + var imageV1Alpha1Compatible bool + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + globalExtraConfig = make(map[string]string) + classConfigSpec = nil + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecExtraConfig( + config, + configSpec, + classConfigSpec, + vmClassSpec, + vm, + globalExtraConfig, + imageV1Alpha1Compatible) + + ecMap = make(map[string]string) + for _, ec := range configSpec.ExtraConfig { + if optionValue := ec.GetOptionValue(); optionValue != nil { + ecMap[optionValue.Key] = optionValue.Value.(string) + } + } + }) + + Context("Empty input", func() { + It("No changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("Updates configSpec.ExtraConfig", func() { + BeforeEach(func() { + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{ + Key: constants.VMOperatorV1Alpha1ExtraConfigKey, Value: constants.VMOperatorV1Alpha1ConfigReady}) + globalExtraConfig["guestinfo.test"] = "test" + globalExtraConfig["global"] = "test" + imageV1Alpha1Compatible = true + }) + + It("Expected configSpec.ExtraConfig", func() { + By("VM Image compatible", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + + By("Global map", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.test", "test")) + Expect(ecMap).To(HaveKeyWithValue("global", "test")) + }) + }) + + Context("When VM uses metadata transport types other than CloudInit", func() { + BeforeEach(func() { + vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + }) + It("defer cloud-init extra config is enabled", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + }) + + Context("When VM uses CloudInit metadata transport type", func() { + BeforeEach(func() { + vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + }) + It("defer cloud-init extra config is not enabled", func() { + Expect(ecMap).ToNot(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + }) + }) + + Context("ExtraConfig value already exists", func() { + BeforeEach(func() { + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{Key: "foo", Value: "bar"}) + globalExtraConfig["foo"] = "bar" + }) + + It("No changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("InstanceStorage related tests", func() { + + Context("When InstanceStorage is NOT configured on VM", func() { + It("No Changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("When InstanceStorage is configured on VM", func() { + BeforeEach(func() { + vm.Spec.Volumes = append(vm.Spec.Volumes, vmopv1.VirtualMachineVolume{ + Name: "pvc-volume-1", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-volume-1", + }, + InstanceVolumeClaim: &vmopv1.InstanceVolumeClaimVolumeSource{ + StorageClass: "dummyStorageClass", + Size: resource.MustParse("256Gi"), + }, + }, + }, + }) + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + }) + }) + + Context("ThunderPciDevices related test", func() { + + Context("when virtual devices are not present", func() { + It("No Changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("when vGPU device is available", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Devices = vmopv1.VirtualDevices{VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "test-vgpu-profile", + }, + }} + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + + It("PCI passthru MMIO extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + + Context("when PCI passthru MMIO override annotation is set", func() { + BeforeEach(func() { + vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] = "12345" + }) + + It("PCI passthru MMIO extraConfig should be set to override annotation value", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, "12345")) + }) + }) + }) + + Context("when DDPIO device is available", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Devices = vmopv1.VirtualDevices{DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 123, + DeviceID: 24, + CustomLabel: "", + }, + }} + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + + It("PCI passthru MMIO extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + + Context("when PCI passthru MMIO override annotation is set", func() { + BeforeEach(func() { + vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] = "12345" + }) + + It("PCI passthru MMIO extraConfig should be set to override annotation value", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, "12345")) + }) + }) + }) + }) + + Context("when VM_Class_as_Config_DaynDate FSS is enabled", func() { + var oldVMClassAsConfigDaynDateFunc func() bool + const dummyKey = "dummy-key" + const dummyVal = "dummy-val" + + BeforeEach(func() { + oldVMClassAsConfigDaynDateFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigDaynDateFunc + }) + + Context("classConfigSpec extra config is not nil", func() { + BeforeEach(func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ExtraConfig: []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{ + Key: dummyKey + "-1", + Value: dummyVal + "-1", + }, + &vimTypes.OptionValue{ + Key: dummyKey + "-2", + Value: dummyVal + "-2", + }, + }, + } + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{Key: "hello", Value: "world"}) + }) + It("vm extra config overlaps with global extra config", func() { + globalExtraConfig["hello"] = "world" + + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-1", dummyVal+"-1")) + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-2", dummyVal+"-2")) + Expect(ecMap).ToNot(HaveKeyWithValue("hello", "world")) + }) + + It("global extra config overlaps with class config spec - class config spec takes precedence", func() { + globalExtraConfig[dummyKey+"-1"] = dummyVal + "-3" + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-1", dummyVal+"-1")) + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-2", dummyVal+"-2")) + }) + + Context("class config spec has vGPU and DDPIO devices", func() { + BeforeEach(func() { + classConfigSpec.DeviceChange = []vimTypes.BaseVirtualDeviceConfigSpec{ + &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "SampleLabel2", + }, + }, + }, + }, + } + + }) + + It("extraConfig Map has MMIO and MMPowerOff related keys added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + }) + }) + }) + }) + + Context("ChangeBlockTracking", func() { + var vmSpec vmopv1.VirtualMachineSpec + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + + BeforeEach(func() { + config.ChangeTrackingEnabled = nil + classConfigSpec = nil + }) + + AfterEach(func() { + configSpec.ChangeTrackingEnabled = nil + }) + + It("cbt and status cbt unset", func() { + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("configSpec cbt set to true OMG", func() { + config.ChangeTrackingEnabled = pointer.Bool(true) + vmSpec.Advanced.ChangeBlockTracking = false + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeFalse()) + }) + + It("configSpec cbt set to false", func() { + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + + It("configSpec cbt matches", func() { + config.ChangeTrackingEnabled = pointer.Bool(true) + vmSpec.Advanced.ChangeBlockTracking = true + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("classConfigSpec not nil and is ignored", func() { + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(false), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + + Context("VM_Class_as_Config_DaynDate FSS is enabled", func() { + var oldVMClassAsConfigDaynDateFunc func() bool + BeforeEach(func() { + oldVMClassAsConfigDaynDateFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigDaynDateFunc + }) + + It("classConfigSpec not nil and same as configInfo", func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(false), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("classConfigSpec not nil, different from configInfo, overrides vm spec cbt", func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(true), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + }) + }) + + Context("Firmware", func() { + var vm *vmopv1.VirtualMachine + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + config.Firmware = "bios" + }) + + It("No firmware annotation", func() { + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + + It("Set firmware annotation equal to current vm firmware", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = config.Firmware + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + + It("Set firmware annotation differing to current vm firmware", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "efi" + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(Equal("efi")) + }) + + It("Set firmware annotation to an invalid value", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "invalidfirmware" + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + }) + + Context("DeviceGroups", func() { + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + + BeforeEach(func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{} + }) + + It("No DeviceGroups set in class config spec", func() { + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).To(BeNil()) + }) + + It("DeviceGroups set in class config spec", func() { + classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(400), + }, + }, + } + + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).NotTo(BeNil()) + Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) + deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() + Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) + }) + + It("configInfo DeviceGroups set with vals different than the class config spec", func() { + classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(400), + }, + }, + } + + config.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(500), + }, + }, + } + + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).NotTo(BeNil()) + Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) + deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() + Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) + }) + }) + + Context("Ethernet Card Changes", func() { + var expectedList object.VirtualDeviceList + var currentList object.VirtualDeviceList + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + var dvpg1 *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo + var dvpg2 *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo + var err error + + BeforeEach(func() { + dvpg1 = &vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo{ + Port: vimTypes.DistributedVirtualSwitchPortConnection{ + PortgroupKey: "key1", + SwitchUuid: "uuid1", + }, + } + + dvpg2 = &vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo{ + Port: vimTypes.DistributedVirtualSwitchPortConnection{ + PortgroupKey: "key2", + SwitchUuid: "uuid2", + }, + } + }) + + JustBeforeEach(func() { + deviceChanges, err = session.UpdateEthCardDeviceChanges(expectedList, currentList) + }) + + AfterEach(func() { + currentList = nil + expectedList = nil + }) + + Context("No devices", func() { + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + + Context("No device change when nothing changes", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + It("returns no device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(0)) + }) + }) + + Context("Add device", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + }) + + It("returns add device change", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(1)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when backing change", func() { + var card1 vimTypes.BaseVirtualDevice + var card2 vimTypes.BaseVirtualDevice + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg2) + Expect(err).ToNot(HaveOccurred()) + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when MAC address is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = "mac1" + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = "mac2" + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("When WCP_VMClass_as_Config is enabled, Add and remove device when card type is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + var oldVMClassAsConfigFunc func() bool + + BeforeEach(func() { + oldVMClassAsConfigFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet2", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigFunc + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when ExternalID is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = "ext1" + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = "ext2" + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Keeps existing device with same backing", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + }) + + Context("Create vSphere PCI device", func() { + var vgpuDevices = []vmopv1.VGPUDevice{ + { + ProfileName: "SampleProfile", + }, + } + var ddpioDevices = []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 42, + DeviceID: 43, + CustomLabel: "SampleLabel", + }, + } + var pciDevices vmopv1.VirtualDevices + Context("For VM Class Spec vGPU device", func() { + BeforeEach(func() { + pciDevices = vmopv1.VirtualDevices{ + VGPUDevices: vgpuDevices, + } + }) + It("should create vSphere device with VmiopBackingInfo", func() { + vSphereDevices := virtualmachine.CreatePCIDevicesFromVMClass(pciDevices) + Expect(vSphereDevices).To(HaveLen(1)) + virtualDevice := vSphereDevices[0].GetVirtualDevice() + backing := virtualDevice.Backing.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo) + Expect(backing.Vgpu).To(Equal(pciDevices.VGPUDevices[0].ProfileName)) + }) + }) + Context("For VM Class Spec Dynamic DirectPath I/O device", func() { + BeforeEach(func() { + pciDevices = vmopv1.VirtualDevices{ + DynamicDirectPathIODevices: ddpioDevices, + } + }) + It("should create vSphere device with DynamicBackingInfo", func() { + vSphereDevices := virtualmachine.CreatePCIDevicesFromVMClass(pciDevices) + Expect(vSphereDevices).To(HaveLen(1)) + virtualDevice := vSphereDevices[0].GetVirtualDevice() + backing := virtualDevice.Backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo) + Expect(backing.AllowedDevice[0].DeviceId).To(Equal(int32(pciDevices.DynamicDirectPathIODevices[0].DeviceID))) + Expect(backing.AllowedDevice[0].VendorId).To(Equal(int32(pciDevices.DynamicDirectPathIODevices[0].VendorID))) + Expect(backing.CustomLabel).To(Equal(pciDevices.DynamicDirectPathIODevices[0].CustomLabel)) + }) + }) + + When("PCI devices from ConfigSpec are specified", func() { + + var devIn []*vimTypes.VirtualPCIPassthrough + + Context("For ConfigSpec VGPU device", func() { + BeforeEach(func() { + devIn = []*vimTypes.VirtualPCIPassthrough{ + { + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "configspec-profile", + }, + }, + }, + } + }) + It("should create vSphere device with VmiopBackingInfo", func() { + devList := virtualmachine.CreatePCIDevicesFromConfigSpec(devIn) + Expect(devList).To(HaveLen(1)) + + Expect(devList[0]).ToNot(BeNil()) + Expect(devList[0]).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthrough{})) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).ToNot(BeNil()) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthroughVmiopBackingInfo{})) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo).Vgpu).To(Equal("configspec-profile")) + }) + }) + + Context("For ConfigSpec DirectPath I/O device", func() { + BeforeEach(func() { + devIn = []*vimTypes.VirtualPCIPassthrough{ + { + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + CustomLabel: "configspec-ddpio-label", + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 456, + DeviceId: 457, + }, + }, + }, + }, + }, + } + }) + It("should create vSphere device with DynamicBackingInfo", func() { + devList := virtualmachine.CreatePCIDevicesFromConfigSpec(devIn) + Expect(devList).To(HaveLen(1)) + + Expect(devList[0]).ToNot(BeNil()) + Expect(devList[0]).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthrough{})) + + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).ToNot(BeNil()) + backing := devList[0].(*vimTypes.VirtualPCIPassthrough).Backing + Expect(backing).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthroughDynamicBackingInfo{})) + + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).CustomLabel).To(Equal("configspec-ddpio-label")) + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).AllowedDevice[0].VendorId).To(BeEquivalentTo(456)) + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).AllowedDevice[0].DeviceId).To(BeEquivalentTo(457)) + }) + }) + }) + }) + + Context("PCI Device Changes", func() { + var ( + currentList, expectedList object.VirtualDeviceList + deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + err error + + // Variables related to vGPU devices. + backingInfo1, backingInfo2 *vimTypes.VirtualPCIPassthroughVmiopBackingInfo + deviceKey1, deviceKey2 int32 + vGPUDevice1, vGPUDevice2 vimTypes.BaseVirtualDevice + + // Variables related to dynamicDirectPathIO devices. + allowedDev1, allowedDev2 vimTypes.VirtualPCIPassthroughAllowedDevice + backingInfo3, backingInfo4 *vimTypes.VirtualPCIPassthroughDynamicBackingInfo + deviceKey3, deviceKey4 int32 + dynamicDirectPathIODev1, dynamicDirectPathIODev2 vimTypes.BaseVirtualDevice + ) + + BeforeEach(func() { + backingInfo1 = &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{Vgpu: "mockup-vmiop1"} + backingInfo2 = &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{Vgpu: "mockup-vmiop2"} + deviceKey1 = int32(-200) + deviceKey2 = int32(-201) + vGPUDevice1 = virtualmachine.CreatePCIPassThroughDevice(deviceKey1, backingInfo1) + vGPUDevice2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey2, backingInfo2) + + allowedDev1 = vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: 1000, + DeviceId: 100, + } + allowedDev2 = vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: 2000, + DeviceId: 200, + } + backingInfo3 = &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev1}, + CustomLabel: "sampleLabel3", + } + backingInfo4 = &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev2}, + CustomLabel: "sampleLabel4", + } + deviceKey3 = int32(-202) + deviceKey4 = int32(-203) + dynamicDirectPathIODev1 = virtualmachine.CreatePCIPassThroughDevice(deviceKey3, backingInfo3) + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfo4) + }) + + JustBeforeEach(func() { + deviceChanges, err = session.UpdatePCIDeviceChanges(expectedList, currentList) + }) + + AfterEach(func() { + currentList = nil + expectedList = nil + }) + + Context("No devices", func() { + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + + Context("Adding vGPU and dynamicDirectPathIO devices with different backing info", func() { + BeforeEach(func() { + expectedList = append(expectedList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice2) + expectedList = append(expectedList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(len(expectedList))) + + for idx, dev := range deviceChanges { + configSpec := dev.GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[idx].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("Adding vGPU and dynamicDirectPathIO devices with same backing info", func() { + BeforeEach(func() { + expectedList = append(expectedList, vGPUDevice1) + // Creating a vGPUDevice with same backingInfo1 but different deviceKey. + vGPUDevice2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey2, backingInfo1) + expectedList = append(expectedList, vGPUDevice2) + expectedList = append(expectedList, dynamicDirectPathIODev1) + // Creating a dynamicDirectPathIO device with same backingInfo3 but different deviceKey. + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfo3) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(len(expectedList))) + + for idx, dev := range deviceChanges { + configSpec := dev.GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[idx].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("When the expected and current lists have DDPIO devices with different custom labels", func() { + BeforeEach(func() { + expectedList = []vimTypes.BaseVirtualDevice{dynamicDirectPathIODev1} + // Creating a dynamicDirectPathIO device with same backing info except for the custom label. + backingInfoDiffCustomLabel := &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: backingInfo3.AllowedDevice, + CustomLabel: "DifferentLabel", + } + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfoDiffCustomLabel) + currentList = []vimTypes.BaseVirtualDevice{dynamicDirectPathIODev2} + }) + + It("should return add and remove device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(currentList[0].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[0].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("When the expected and current list of pciDevices have different Devices", func() { + BeforeEach(func() { + currentList = append(currentList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice2) + currentList = append(currentList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add and remove device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(4)) + + for i := 0; i < 2; i++ { + configSpec := deviceChanges[i].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(currentList[i].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + } + + for i := 2; i < 4; i++ { + configSpec := deviceChanges[i].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[i-2].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("When the expected and current list of pciDevices have same Devices", func() { + BeforeEach(func() { + currentList = append(currentList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice1) + currentList = append(currentList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev1) + }) + + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/storage/provisioning.go b/pkg/vmprovider/providers/vsphere2/storage/provisioning.go new file mode 100644 index 000000000..5a3997a65 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/storage/provisioning.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "github.com/pkg/errors" + "github.com/vmware/govmomi/pbm" + pbmTypes "github.com/vmware/govmomi/pbm/types" + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" +) + +// getProfileProportionalCapacity returns the storage profile "proportionalCapacity" value if the +// policy is for vSAN. The proportionalCapacity is percentage of the logical size of the storage +// object that will be reserved upon provisioning. Returns -1 if not specified. +// The UI presents options for "thin" (0%), 25%, 50%, 75% and "thick" (100%). +func getProfileProportionalCapacity(profile pbmTypes.BasePbmProfile) int32 { + capProfile, ok := profile.(*pbmTypes.PbmCapabilityProfile) + if !ok { + return -1 + } + + if capProfile.ResourceType.ResourceType != string(pbmTypes.PbmProfileResourceTypeEnumSTORAGE) { + return -1 + } + + if capProfile.ProfileCategory != string(pbmTypes.PbmProfileCategoryEnumREQUIREMENT) { + return -1 + } + + sub, ok := capProfile.Constraints.(*pbmTypes.PbmCapabilitySubProfileConstraints) + if !ok { + return -1 + } + + for _, p := range sub.SubProfiles { + for _, capability := range p.Capability { + if capability.Id.Namespace != "VSAN" || capability.Id.Id != "proportionalCapacity" { + continue + } + + for _, c := range capability.Constraint { + for _, prop := range c.PropertyInstance { + if prop.Id != capability.Id.Id { + continue + } + if val, ok := prop.Value.(int32); ok { + return val + } + } + } + } + } + + return -1 +} + +// GetDiskProvisioningForProfile returns the provisioning type for the storage profile if it has +// one specified. +func GetDiskProvisioningForProfile( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + storageProfileID string) (string, error) { + + c, err := pbm.NewClient(vmCtx, vcClient.VimClient()) + if err != nil { + return "", err + } + + profiles, err := c.RetrieveContent(vmCtx, []pbmTypes.PbmProfileId{{UniqueId: storageProfileID}}) + if err != nil { + return "", errors.Wrapf(err, "Failed to get storage profiles for ID: %s", storageProfileID) + } + + for _, p := range profiles { + switch getProfileProportionalCapacity(p) { + case 0: + return string(vimTypes.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil + case 100: + return string(vimTypes.OvfCreateImportSpecParamsDiskProvisioningTypeThick), nil + } + } + + return "", nil +} diff --git a/pkg/vmprovider/providers/vsphere2/storage/storageclass.go b/pkg/vmprovider/providers/vsphere2/storage/storageclass.go new file mode 100644 index 000000000..511e651a0 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/storage/storageclass.go @@ -0,0 +1,83 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + + storagev1 "k8s.io/api/storage/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +// GetStoragePolicyID returns Storage Policy ID from Storage Class Name. +func GetStoragePolicyID( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + storageClassName string) (string, error) { + + sc := &storagev1.StorageClass{} + if err := client.Get(vmCtx, ctrlclient.ObjectKey{Name: storageClassName}, sc); err != nil { + vmCtx.Logger.Error(err, "Failed to get StorageClass", "storageClass", storageClassName) + return "", err + } + + policyID, ok := sc.Parameters["storagePolicyID"] + if !ok { + return "", fmt.Errorf("StorageClass %s does not have 'storagePolicyID' parameter", storageClassName) + } + + return policyID, nil +} + +// GetVMStoragePoliciesIDs returns a map of storage class names to their storage policy IDs. +func GetVMStoragePoliciesIDs( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client) (map[string]string, error) { + + storageClassNames := getVMStorageClassNames(vmCtx.VM) + storageClassesToIDs := map[string]string{} + + for _, name := range storageClassNames { + if _, ok := storageClassesToIDs[name]; !ok { + id, err := GetStoragePolicyID(vmCtx, client, name) + if err != nil { + return nil, err + } + + storageClassesToIDs[name] = id + } + } + + return storageClassesToIDs, nil +} + +func getVMStorageClassNames(vm *vmopv1.VirtualMachine) []string { + var names []string + + if vm.Spec.StorageClass != "" { + names = append(names, vm.Spec.StorageClass) + } + + for _, vol := range vm.Spec.Volumes { + var storageClass string + + claim := vol.PersistentVolumeClaim + if claim != nil { + if isClaim := claim.InstanceVolumeClaim; isClaim != nil { + storageClass = isClaim.StorageClass + } else { //nolint + // TODO: Fetch claim.ClaimName PVC to get the StorageClass. + } + } + + if storageClass != "" { + names = append(names, storageClass) + } + } + + return names +} diff --git a/pkg/vmprovider/providers/vsphere2/test/pki.go b/pkg/vmprovider/providers/vsphere2/test/pki.go new file mode 100644 index 000000000..557569440 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/pki.go @@ -0,0 +1,70 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "time" + + . "github.com/onsi/gomega" +) + +func GeneratePrivateKey() *rsa.PrivateKey { + reader := rand.Reader + bitSize := 2048 + + // Based on https://golang.org/src/crypto/tls/generate_cert.go + privateKey, err := rsa.GenerateKey(reader, bitSize) + if err != nil { + panic("failed to generate private key") + } + return privateKey +} + +func GenerateSelfSignedCert() (string, string) { + priv := GeneratePrivateKey() + now := time.Now() + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + Expect(err).NotTo(HaveOccurred()) + + template := x509.Certificate{ + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + }, + SerialNumber: serialNumber, + NotBefore: now, + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + template.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + Expect(err).NotTo(HaveOccurred()) + certOut, err := os.CreateTemp("", "cert.pem") + Expect(err).NotTo(HaveOccurred()) + err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + Expect(err).NotTo(HaveOccurred()) + err = certOut.Close() + Expect(err).NotTo(HaveOccurred()) + + keyOut, err := os.CreateTemp("", "key.pem") + Expect(err).NotTo(HaveOccurred()) + privBytes, err := x509.MarshalPKCS8PrivateKey(priv) + Expect(err).NotTo(HaveOccurred()) + err = pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + Expect(err).NotTo(HaveOccurred()) + err = keyOut.Close() + Expect(err).NotTo(HaveOccurred()) + + return keyOut.Name(), certOut.Name() +} diff --git a/pkg/vmprovider/providers/vsphere2/test/suite.go b/pkg/vmprovider/providers/vsphere2/test/suite.go new file mode 100644 index 000000000..cf496c076 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/suite.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "context" + "crypto/tls" + "os" + + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + _ "github.com/vmware/govmomi/vapi/simulator" // blank import the VAPI simulator bindings +) + +func BeforeSuite() (ctx context.Context, + model *simulator.Model, + server *simulator.Server, + tlsKeyPath, tlsCertPath string, + tlsModel *simulator.Model, + tlsServer *simulator.Server) { + + ctx = context.Background() + + // Set up a simulator for testing most client interactions (ignoring TLS) + model, server = SetupModelAndServerWithSettings(&tls.Config{ + MinVersion: tls.VersionTLS12, + }) + + // Set up a second simulator for testing TLS. + tlsKeyPath, tlsCertPath = GenerateSelfSignedCert() + tlsCert, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) + Expect(err).NotTo(HaveOccurred()) + tlsModel, tlsServer = SetupModelAndServerWithSettings(&tls.Config{ + Certificates: []tls.Certificate{ + tlsCert, + }, + PreferServerCipherSuites: true, + MinVersion: tls.VersionTLS12, + }) + + return +} + +func AfterSuite( + ctx context.Context, + model *simulator.Model, + server *simulator.Server, + tlsKeyPath, tlsCertPath string, + tlsModel *simulator.Model, + tlsServer *simulator.Server) { + + server.Close() + model.Remove() + + tlsServer.Close() + tlsModel.Remove() + + _ = os.Remove(tlsKeyPath) + _ = os.Remove(tlsCertPath) +} diff --git a/pkg/vmprovider/providers/vsphere2/test/vcsim.go b/pkg/vmprovider/providers/vsphere2/test/vcsim.go new file mode 100644 index 000000000..607ed81e2 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/vcsim.go @@ -0,0 +1,31 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "crypto/tls" + + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" +) + +func SetupModelAndServerWithSettings(tlsConfig *tls.Config) (*simulator.Model, *simulator.Server) { + newModel := simulator.VPX() + + // By Default, the Model being used by vcsim has two ResourcePools + // (one for the cluster and host each). Setting Model.Host=0 ensures + // we only have one ResourcePool, making it easier to pick the + // ResourcePool without having to look up using a hardcoded path. + newModel.Host = 0 + + err := newModel.Create() + Expect(err).ToNot(HaveOccurred()) + + newModel.Service.RegisterEndpoints = true + + newModel.Service.TLS = tlsConfig + newServer := newModel.Service.NewServer() + + return newModel, newServer +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go b/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go new file mode 100644 index 000000000..2cbee7adb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25/mo" +) + +// ClusterMinCPUFreq returns the minimum frequency across all the hosts in the cluster. This is needed to +// convert the CPU requirements specified in cores to MHz. vSphere core is assumed to be equivalent to the +// value of min frequency. This function is adapted from wcp schedext. +func ClusterMinCPUFreq(ctx goctx.Context, cluster *object.ClusterComputeResource) (uint64, error) { + var cr mo.ComputeResource + if err := cluster.Properties(ctx, cluster.Reference(), []string{"host"}, &cr); err != nil { + return 0, err + } + + if len(cr.Host) == 0 { + return 0, nil + } + + var hosts []mo.HostSystem + pc := property.DefaultCollector(cluster.Client()) + if err := pc.Retrieve(ctx, cr.Host, []string{"summary"}, &hosts); err != nil { + return 0, err + } + + var minFreq uint64 + for _, h := range hosts { + if hw := h.Summary.Hardware; hw != nil { + hostCPUMHz := uint64(hw.CpuMhz) + if hostCPUMHz < minFreq || minFreq == 0 { + minFreq = hostCPUMHz + } + } + } + + return minFreq, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go new file mode 100644 index 000000000..5b90b5145 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func clusterTests() { + Describe("ClusterMinCPUFreq", minFreq) +} + +func minFreq() { + // Hardcoded value in govmomi simulator/esx/host_system.go + const expectedCPUFreq = 2294 + + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("ClusterMinCPUFreq", func() { + It("returns min freq of hosts in cluster", func() { + cpuFreq, err := vcenter.ClusterMinCPUFreq(ctx, ctx.GetSingleClusterCompute()) + Expect(err).ToNot(HaveOccurred()) + Expect(cpuFreq).Should(BeEquivalentTo(expectedCPUFreq)) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/folder.go b/pkg/vmprovider/providers/vsphere2/vcenter/folder.go new file mode 100644 index 000000000..414a08a67 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/folder.go @@ -0,0 +1,138 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" +) + +// GetFolderByMoID returns the vim Folder for the MoID. +func GetFolderByMoID( + ctx goctx.Context, + finder *find.Finder, + folderMoID string) (*object.Folder, error) { + + o, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "Folder", Value: folderMoID}) + if err != nil { + return nil, err + } + + return o.(*object.Folder), nil +} + +// GetChildFolder gets the named child Folder from the parent Folder. +func GetChildFolder( + ctx goctx.Context, + parentFolder *object.Folder, + childName string) (*object.Folder, error) { + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return nil, err + } else if childFolder == nil { + return nil, fmt.Errorf("folder child %s not found under parent Folder %s", + childName, parentFolder.Reference().Value) + } + + return childFolder, nil +} + +// DoesChildFolderExist returns if the named child Folder exists under the parent Folder. +func DoesChildFolderExist( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) (bool, error) { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return false, err + } + + return childFolder != nil, nil +} + +// CreateFolder creates the named child Folder under the parent Folder. +func CreateFolder( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) (string, error) { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return "", err + } + + if childFolder == nil { + folder, err := parentFolder.CreateFolder(ctx, childName) + if err != nil { + return "", err + } + + childFolder = folder + } + + return childFolder.Reference().Value, nil +} + +// DeleteChildFolder deletes the child Folder under the parent Folder. +func DeleteChildFolder( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) error { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil || childFolder == nil { + return err + } + + task, err := childFolder.Destroy(ctx) + if err != nil { + return err + } + + if taskResult, err := task.WaitForResult(ctx); err != nil { + if taskResult == nil || taskResult.Error == nil { + return err + } + return fmt.Errorf("destroy Folder %s task failed: %w: %s", + childFolder.Reference().Value, err, taskResult.Error.LocalizedMessage) + } + + return nil +} + +func findChildFolder( + ctx goctx.Context, + parentFolder *object.Folder, + childName string) (*object.Folder, error) { + + objRef, err := object.NewSearchIndex(parentFolder.Client()).FindChild(ctx, parentFolder.Reference(), childName) + if err != nil { + return nil, err + } else if objRef == nil { + return nil, nil + } + + folder, ok := objRef.(*object.Folder) + if !ok { + return nil, fmt.Errorf("Folder child %q is not Folder but a %T", childName, objRef) //nolint + } + + return folder, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go new file mode 100644 index 000000000..99cfa9426 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go @@ -0,0 +1,173 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func folderTests() { + Describe("GetFolderByMoID", getFolderByMoID) + Describe("CreateDeleteExistsFolder", createDeleteExistsFolder) +} + +func getFolderByMoID() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("returns success", func() { + moID := nsInfo.Folder.Reference().Value + + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, moID) + Expect(err).ToNot(HaveOccurred()) + Expect(folder).ToNot(BeNil()) + Expect(folder.Name()).To(Equal(nsInfo.Namespace)) + }) + + It("returns error when moID does not exist", func() { + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, "bogus") + Expect(err).To(HaveOccurred()) + Expect(folder).To(BeNil()) + }) +} + +func createDeleteExistsFolder() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + + parentFolderMoID string + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + parentFolderMoID = nsInfo.Folder.Reference().Value + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + parentFolderMoID = "" + }) + + Context("CreateFolder", func() { + It("creates child Folder", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(childMoID).ToNot(BeEmpty()) + + By("NoOp when child Folder already exists", func() { + moID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(moID).To(Equal(childMoID)) + }) + + By("child Folder is found by MoID", func() { + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, childMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(folder.Reference().Value).To(Equal(childMoID)) + }) + }) + + It("returns error when parent Folder MoID does not exist", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, "bogus", "myFolder") + Expect(err).To(HaveOccurred()) + Expect(childMoID).To(BeEmpty()) + }) + }) + + Context("GetChildFolder", func() { + It("returns success when child Folder exists", func() { + childFolderMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + parentFolder := object.NewFolder(ctx.VCClient.Client, types.ManagedObjectReference{ + Type: "Folder", + Value: parentFolderMoID, + }) + + childFolder, err := vcenter.GetChildFolder(ctx, parentFolder, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(childFolder).ToNot(BeNil()) + Expect(childFolder.Reference().Value).To(Equal(childFolderMoID)) + }) + + It("returns error when child Folder does not exists", func() { + parentFolder := object.NewFolder(ctx.VCClient.Client, types.ManagedObjectReference{ + Type: "Folder", + Value: parentFolderMoID, + }) + + childFolder, err := vcenter.GetChildFolder(ctx, parentFolder, "myFolder") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found under parent Folder")) + Expect(childFolder).To(BeNil()) + }) + }) + + Context("DoesChildFolderExist", func() { + It("returns true when child Folder exists", func() { + _, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("returns false when child Folder does not exist", func() { + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + It("returns error when parent Folder MoID does not exist", func() { + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, "bogus", "myFolder") + Expect(err).To(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("DeleteFolder", func() { + It("deletes child Folder", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + err = vcenter.DeleteChildFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + By("child Folder is not found by MoID", func() { + _, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, childMoID) + Expect(err).To(HaveOccurred()) + }) + + By("NoOp when child does not exist", func() { + err := vcenter.DeleteChildFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go b/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go new file mode 100644 index 000000000..f6363a31c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go @@ -0,0 +1,152 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" +) + +// GetVirtualMachine gets the VM from VC, either by the MoID, UUID, or the inventory path. +func GetVirtualMachine( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vimClient *vim25.Client, + datacenter *object.Datacenter, + finder *find.Finder) (*object.VirtualMachine, error) { + + if uniqueID := vmCtx.VM.Status.UniqueID; uniqueID != "" { + if vm, err := findVMByMoID(vmCtx, finder, uniqueID); err == nil { + return vm, nil + } + } + + // For when we start to use the k8s VM.UID for the VC VM's InstanceUUID or UUID (aka BiosUUID): + /* + if instanceUUID := vmCtx.VM.UID; instanceUUID != "" { + if vm, err := findVMByUUID(vmCtx, vimClient, datacenter, string(instanceUUID), true); err == nil { + return vm, nil + } + } + */ + + return findVMByInventory(vmCtx, k8sClient, vimClient, finder) +} + +func findVMByMoID( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + moID string) (*object.VirtualMachine, error) { + + ref, err := finder.ObjectReference(vmCtx, types.ManagedObjectReference{Type: "VirtualMachine", Value: moID}) + if err != nil { + return nil, err + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VM but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via MoID", "path", vm.InventoryPath, "moID", moID) + return vm, nil +} + +//nolint:unused +func findVMByUUID( + vmCtx context.VirtualMachineContextA2, + vimClient *vim25.Client, + datacenter *object.Datacenter, + uuid string, + isInstanceUUID bool) (*object.VirtualMachine, error) { + + ref, err := object.NewSearchIndex(vimClient).FindByUuid(vmCtx, datacenter, uuid, true, &isInstanceUUID) + if err != nil { + return nil, fmt.Errorf("error finding object by UUID %q: %w", uuid, err) + } else if ref == nil { + return nil, fmt.Errorf("no VM found for UUID %q (instanceUUID: %v)", uuid, isInstanceUUID) + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VirtualMachine but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via UUID", "uuid", uuid, "isInstanceUUID", isInstanceUUID) + return vm, nil +} + +func findVMByInventory( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vimClient *vim25.Client, + finder *find.Finder) (*object.VirtualMachine, error) { + + // Note that we'll usually only get here to find the VM via its inventory path when we're first + // creating the VM. To determine the path, we need the NS Folder MoID and the VM's ResourcePolicy, + // if set, and we'll fetch these again as a part of createVirtualMachine(). For now, just re-fetch + // but we could pass the Folder MoID and ResourcePolicy to save a bit of duplicated work. + + folderMoID, err := topology.GetNamespaceFolderMoID(vmCtx, k8sClient, vmCtx.VM.Namespace) + if err != nil { + return nil, err + } + + // While we strictly only need the Folder's ManagedObjectReference below, use the Finder + // here to check if it actually exists. + folder, err := GetFolderByMoID(vmCtx, finder, folderMoID) + if err != nil { + return nil, fmt.Errorf("failed to get namespace Folder: %w", err) + } + + // When the VM has a ResourcePolicy, the VM is placed in a child folder under the namespace's folder. + if policyName := vmCtx.VM.Spec.Reserved.ResourcePolicyName; policyName != "" { + resourcePolicy := &vmopv1.VirtualMachineSetResourcePolicy{} + + key := ctrlclient.ObjectKey{Name: policyName, Namespace: vmCtx.VM.Namespace} + if err := k8sClient.Get(vmCtx, key, resourcePolicy); err != nil { + // Note that if VM does not exist, and we're about to create it, the ResourcePolicy is Get() + // again so the corresponding condition is almost always true if we don't hit an error here. + // Creating the VM with an explicit InstanceUUID is the easiest way out to avoid this. + return nil, fmt.Errorf("failed to get VirtualMachineSetResourcePolicy: %w", err) + } + + if folderName := resourcePolicy.Spec.Folder; folderName != "" { + childFolder, err := GetChildFolder(vmCtx, folder, folderName) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VirtualMachineSetResourcePolicy child Folder", + "parentPath", folder.InventoryPath, "folderName", folderName, "policyName", policyName) + return nil, err + } + + folder = childFolder + } + } + + ref, err := object.NewSearchIndex(vimClient).FindChild(vmCtx, folder.Reference(), vmCtx.VM.Name) + if err != nil { + return nil, err + } else if ref == nil { + // VM does not exist. + return nil, nil + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VM but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via inventory", + "parentFolderMoID", folder.Reference().Value, "moID", vm.Reference().Value) + return vm, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go new file mode 100644 index 000000000..29b48be5a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go @@ -0,0 +1,158 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/apimachinery/pkg/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func getVMTests() { + Describe("GetVirtualMachine", getVM) +} + +func getVM() { + // Use a VM that vcsim creates for us. + const vcVMName = "DC0_C0_RP0_VM0" + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + + vm := builder.DummyVirtualMachineA2() + vm.Name = "getvm-test" + vm.Namespace = nsInfo.Namespace + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + Context("Gets VM by inventory", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + task, err := vm.Clone(ctx, nsInfo.Folder, vmCtx.VM.Name, vimtypes.VirtualMachineCloneSpec{}) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + + It("returns nil if VM does not exist", func() { + vmCtx.VM.Name = "bogus" + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).To(BeNil()) + }) + + Context("Namespace Folder does not exist", func() { + BeforeEach(func() { + task, err := nsInfo.Folder.Destroy(vmCtx) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(vmCtx)).To(Succeed()) + }) + + It("returns error", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to get namespace Folder")) + Expect(vm).To(BeNil()) + }) + }) + + It("returns success when MoID is invalid", func() { + // Expect fallback to inventory. + vmCtx.VM.Status.UniqueID = "vm-bogus" + + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + }) + + Context("Gets VM when MoID is set", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + vmCtx.VM.Status.UniqueID = vm.Reference().Value + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + Expect(vm.Reference().Value).To(Equal(vmCtx.VM.Status.UniqueID)) + }) + }) + + // Not until we start setting either the InstanceUUID or BiosUUID + XContext("Gets VM by UUID", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vm.Properties(ctx, vm.Reference(), nil, &o)).To(Succeed()) + vmCtx.VM.UID = types.UID(o.Config.InstanceUuid) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + }) + + Context("Gets VM with ResourcePolicy by inventory", func() { + BeforeEach(func() { + resourcePolicy, folder := ctx.CreateVirtualMachineSetResourcePolicyA2("getvm-test", nsInfo) + vmCtx.VM.Spec.Reserved.ResourcePolicyName = resourcePolicy.Name + + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + task, err := vm.Clone(ctx, folder, vmCtx.VM.Name, vimtypes.VirtualMachineCloneSpec{}) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + + It("returns error when ResourcePolicy does not exist", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "bogus" + + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to get VirtualMachineSetResourcePolicy")) + Expect(vm).To(BeNil()) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/host.go b/pkg/vmprovider/providers/vsphere2/vcenter/host.go new file mode 100644 index 000000000..e121a5a45 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/host.go @@ -0,0 +1,41 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "fmt" + "strings" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" +) + +// GetESXHostFQDN returns the ESX host's FQDN. +func GetESXHostFQDN( + ctx context.Context, + vimClient *vim25.Client, + hostMoID string) (string, error) { + + hostMoRef := types.ManagedObjectReference{Type: "HostSystem", Value: hostMoID} + networkSys, err := object.NewHostSystem(vimClient, hostMoRef).ConfigManager().NetworkSystem(ctx) + if err != nil { + return "", fmt.Errorf("failed to get HostNetworkSystem for hostMoID %s: %w", hostMoID, err) + } + + var hostNetworkSys mo.HostNetworkSystem + if err := networkSys.Properties(ctx, networkSys.Reference(), []string{"dnsConfig"}, &hostNetworkSys); err != nil { + return "", fmt.Errorf("failed to get HostMoID %s DNSConfig prop: %w", hostMoID, err) + } + + if hostNetworkSys.DnsConfig == nil { + return "", fmt.Errorf("hostMoID %s HostNetworkSystem does not have DNSConfig", hostMoID) + } + + hostDNSConfig := hostNetworkSys.DnsConfig.GetHostDnsConfig() + hostFQDN := strings.TrimSuffix(hostDNSConfig.HostName+"."+hostDNSConfig.DomainName, ".") + return strings.ToLower(hostFQDN), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go new file mode 100644 index 000000000..9b7fbabdc --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func hostTests() { + Describe("GetESXHostFQDN", hostFQDN) +} + +func hostFQDN() { + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + + hostMoID string + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + testConfig.WithInstanceStorage = true + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + hosts, err := ctx.Finder.HostSystemList(ctx, "*") + Expect(err).ToNot(HaveOccurred()) + Expect(hosts).ToNot(BeEmpty()) + hostMoID = hosts[0].Reference().Value + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("GetESXHostFQDN", func() { + When("host does not have DNSConfig", func() { + BeforeEach(func() { + testConfig.WithInstanceStorage = false + }) + + It("returns expected error", func() { + _, err := vcenter.GetESXHostFQDN(ctx, ctx.VCClient.Client, hostMoID) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(" does not have DNSConfig")) + }) + }) + + It("returns expected host name for host", func() { + hostName, err := vcenter.GetESXHostFQDN(ctx, ctx.VCClient.Client, hostMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(hostName).Should(Equal(fmt.Sprintf("%s.vmop.vmware.com", hostMoID))) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go new file mode 100644 index 000000000..080f1c5af --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go @@ -0,0 +1,163 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +// GetResourcePoolByMoID returns the ResourcePool for the MoID. +func GetResourcePoolByMoID( + ctx goctx.Context, + finder *find.Finder, + rpMoID string) (*object.ResourcePool, error) { + + o, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + if err != nil { + return nil, err + } + + return o.(*object.ResourcePool), nil +} + +// GetResourcePoolOwnerMoRef returns the ClusterComputeResource MoID that owns the ResourcePool. +func GetResourcePoolOwnerMoRef( + ctx goctx.Context, + vimClient *vim25.Client, + rpMoID string) (types.ManagedObjectReference, error) { + + rp := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + objRef, err := rp.Owner(ctx) + if err != nil { + return types.ManagedObjectReference{}, err + } + + return objRef.Reference(), nil +} + +// GetChildResourcePool gets the named child ResourcePool from the parent ResourcePool. +func GetChildResourcePool( + ctx goctx.Context, + parentRP *object.ResourcePool, + childName string) (*object.ResourcePool, error) { + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil { + return nil, err + } else if childRP == nil { + return nil, fmt.Errorf("ResourcePool child %q not found under parent ResourcePool %s", + childName, parentRP.Reference().Value) + } + + return childRP, nil +} + +// DoesChildResourcePoolExist returns if the named child ResourcePool exists under the parent ResourcePool. +func DoesChildResourcePoolExist( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID, childName string) (bool, error) { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil { + return false, err + } + + return childRP != nil, nil +} + +// CreateOrUpdateChildResourcePool creates or updates the child ResourcePool under the parent ResourcePool. +func CreateOrUpdateChildResourcePool( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID string, + rpSpec *vmopv1.ResourcePoolSpec) (string, error) { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, rpSpec.Name) + if err != nil { + return "", err + } + + spec := types.DefaultResourceConfigSpec() // TODO Set reservations & limits from rpSpec + + if childRP == nil { + rp, err := parentRP.Create(ctx, rpSpec.Name, spec) + if err != nil { + return "", err + } + + childRP = rp + } else { //nolint + // TODO: // Finish this clause + } + + return childRP.Reference().Value, nil +} + +// DeleteChildResourcePool deletes the child ResourcePool under the parent ResourcePool. +func DeleteChildResourcePool( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID, childName string) error { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil || childRP == nil { + return err + } + + task, err := childRP.Destroy(ctx) + if err != nil { + return err + } + + if taskResult, err := task.WaitForResult(ctx); err != nil { + if taskResult == nil || taskResult.Error == nil { + return err + } + return fmt.Errorf("destroy ResourcePool %s task failed: %w: %s", + childRP.Reference().Value, err, taskResult.Error.LocalizedMessage) + } + + return nil +} + +func findChildRP( + ctx goctx.Context, + parentRP *object.ResourcePool, + childName string) (*object.ResourcePool, error) { + + objRef, err := object.NewSearchIndex(parentRP.Client()).FindChild(ctx, parentRP, childName) + if err != nil { + return nil, err + } else if objRef == nil { + // FindChild() returns nil when child name is not found. + return nil, nil + } + + childRP, ok := objRef.(*object.ResourcePool) + if !ok { + return nil, fmt.Errorf("ResourcePool child %q is not a ResourcePool but a %T", childName, objRef) + } + + return childRP, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go new file mode 100644 index 000000000..e0e16208d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go @@ -0,0 +1,200 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func resourcePoolTests() { + Describe("GetResourcePool", getResourcePoolTests) + Describe("CreateDeleteExistResourcePoolChild", createDeleteExistResourcePoolChild) +} + +func getResourcePoolTests() { + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + nsRP *object.ResourcePool + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + nsRP = ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + nsRP = nil + }) + + Context("GetResourcePoolByMoID", func() { + It("returns success", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, nsRP.Reference().Value) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).ToNot(BeNil()) + Expect(rp.Reference()).To(Equal(nsRP.Reference())) + }) + + It("returns error when MoID does not exist", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, "bogus") + Expect(err).To(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + }) + + Context("GetResourcePoolOwnerMoRef", func() { + It("returns success", func() { + ccr, err := vcenter.GetResourcePoolOwnerMoRef(ctx, ctx.VCClient.Client, nsRP.Reference().Value) + Expect(err).ToNot(HaveOccurred()) + Expect(ccr).To(Equal(ctx.GetSingleClusterCompute().Reference())) + }) + + It("returns error when MoID does not exist", func() { + _, err := vcenter.GetResourcePoolOwnerMoRef(ctx, ctx.VCClient.Client, "bogus") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("GetChildResourcePool", func() { + It("returns success", func() { + // Quick way for a child RP is to create a VMSetResourcePolicy. + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + + childRP, err := vcenter.GetChildResourcePool(ctx, nsRP, childRPName) + Expect(err).ToNot(HaveOccurred()) + Expect(childRP).ToNot(BeNil()) + + objRef, err := ctx.Finder.ObjectReference(ctx, childRP.Reference()) + Expect(err).ToNot(HaveOccurred()) + childRP, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + Expect(childRP.Name()).To(Equal(resourcePolicy.Name)) + }) + + It("returns error when child RP does not exist", func() { + childRP, err := vcenter.GetChildResourcePool(ctx, nsRP, "bogus") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found under parent ResourcePool")) + Expect(childRP).To(BeNil()) + }) + }) +} + +func createDeleteExistResourcePoolChild() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + nsRP *object.ResourcePool + + parentRPMoID string + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + nsRP = ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + + parentRPMoID = nsRP.Reference().Value + + resourcePolicy, _ = ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + nsRP = nil + parentRPMoID = "" + resourcePolicy = nil + }) + + Context("CreateOrUpdateChildResourcePool", func() { + It("creates child ResourcePool", func() { + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + Expect(childMoID).ToNot(BeEmpty()) + + By("returns success when child ResourcePool already exists", func() { + moID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + Expect(moID).To(Equal(childMoID)) + }) + + By("child ResourcePool is found by MoID", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, childMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(rp.Reference().Value).To(Equal(childMoID)) + }) + }) + + It("returns error when when parent ResourcePool MoID does not exist", func() { + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, "bogus", &resourcePolicy.Spec.ResourcePool) + Expect(err).To(HaveOccurred()) + Expect(childMoID).To(BeEmpty()) + }) + }) + + Context("DoesChildResourcePoolExist", func() { + It("returns true when child ResourcePool exists", func() { + childName := resourcePolicy.Spec.ResourcePool.Name + + _, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("returns false when child ResourcePool does not exist", func() { + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, parentRPMoID, "bogus") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + It("returns error when parent ResourcePool MoID does not exist", func() { + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, "bogus", "bogus") + Expect(err).To(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("DeleteChildResourcePool", func() { + It("deletes child ResourcePool", func() { + childName := resourcePolicy.Spec.ResourcePool.Name + + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + + err = vcenter.DeleteChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + + By("child ResourcePool is not found by MoID", func() { + _, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, childMoID) + Expect(err).To(HaveOccurred()) + }) + + By("NoOp when child does not exist", func() { + err := vcenter.DeleteChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go new file mode 100644 index 000000000..63d223192 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go @@ -0,0 +1,30 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() + +func vcSimTests() { + Describe("Cluster", clusterTests) + Describe("Folder", folderTests) + Describe("GetVM", getVMTests) + Describe("Host", hostTests) + Describe("ResourcePool", resourcePoolTests) +} + +func TestVCenter(t *testing.T) { + suite.Register(t, "VMProvider VCenter Tests", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go new file mode 100644 index 000000000..ff752d426 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go @@ -0,0 +1,34 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + + "github.com/vmware/govmomi/object" +) + +// GetVMClusterComputeResource returns the VM's ClusterComputeResource. +func GetVMClusterComputeResource( + ctx context.Context, + vcVM *object.VirtualMachine) (*object.ClusterComputeResource, error) { + + rp, err := vcVM.ResourcePool(ctx) + if err != nil { + return nil, err + } + + ccrRef, err := rp.Owner(ctx) + if err != nil { + return nil, err + } + + cluster, ok := ccrRef.(*object.ClusterComputeResource) + if !ok { + return nil, fmt.Errorf("VM Owner is not a ClusterComputeResource but %T", ccrRef) + } + + return cluster, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go new file mode 100644 index 000000000..55e4a5842 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func ccrTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Returns VM ClusterComputeResource", func() { + ccr, err := virtualmachine.GetVMClusterComputeResource(ctx, vcVM) + Expect(err).ToNot(HaveOccurred()) + Expect(ccr).ToNot(BeNil()) + Expect(ccr.Reference()).To(Equal(ctx.GetSingleClusterCompute().Reference())) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go new file mode 100644 index 000000000..5ae236f88 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go @@ -0,0 +1,190 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" +) + +// CreateConfigSpec returns an initial ConfigSpec that is created by overlaying the +// base ConfigSpec with VM Class spec and other arguments. +// TODO: We eventually need to de-dupe much of this with the ConfigSpec manipulation that's later done +// in the "update" pre-power on path. That operates on a ConfigInfo so we'd need to populate that from +// the config we build here. +func CreateConfigSpec( + vmCtx context.VirtualMachineContextA2, + vmClassConfigSpec *types.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + vmImageStatus *vmopv1.VirtualMachineImageStatus, + minFreq uint64) *types.VirtualMachineConfigSpec { + + configSpec := types.VirtualMachineConfigSpec{} + + // If there is a class ConfigSpec, then that is our initial ConfigSpec. + if vmClassConfigSpec != nil { + configSpec = *vmClassConfigSpec + } + + configSpec.Name = vmCtx.VM.Name + if configSpec.Annotation == "" { + // If the class ConfigSpec doesn't specify any annotations, set the default one. + configSpec.Annotation = constants.VCVMAnnotation + } + // CPU and Memory configurations specified in the VM Class standalone fields take + // precedence over values in the config spec + configSpec.NumCPUs = int32(vmClassSpec.Hardware.Cpus) + configSpec.MemoryMB = MemoryQuantityToMb(vmClassSpec.Hardware.Memory) + configSpec.ManagedBy = &types.ManagedByInfo{ + ExtensionKey: constants.ManagedByExtensionKey, + Type: constants.ManagedByExtensionType, + } + + if val, ok := vmCtx.VM.Annotations[constants.FirmwareOverrideAnnotation]; ok { + configSpec.Firmware = val + } else if configSpec.Firmware == "" && vmImageStatus != nil { + // Use firmware type from the image if ConfigSpec doesn't have it. + configSpec.Firmware = vmImageStatus.Firmware + } + + // TODO: Otherwise leave as-is? Our ChangeBlockTracking could be better as a *bool. + if vmCtx.VM.Spec.Advanced.ChangeBlockTracking { + configSpec.ChangeTrackingEnabled = pointer.Bool(true) + } + + // Populate the CPU reservation and limits in the ConfigSpec if VAPI fields specify any. + // VM Class VAPI does not support Limits, so they will never be non nil. + // TODO: Remove limits: issues/56 + if res := vmClassSpec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + // TODO: Always override? + configSpec.CpuAllocation = &types.ResourceAllocationInfo{ + Shares: &types.SharesInfo{ + Level: types.SharesLevelNormal, + }, + } + + if !res.Requests.Cpu.IsZero() { + rsv := CPUQuantityToMhz(vmClassSpec.Policies.Resources.Requests.Cpu, minFreq) + configSpec.CpuAllocation.Reservation = &rsv + } + if !res.Limits.Cpu.IsZero() { + lim := CPUQuantityToMhz(vmClassSpec.Policies.Resources.Limits.Cpu, minFreq) + configSpec.CpuAllocation.Limit = &lim + } + } + + // Populate the memory reservation and limits in the ConfigSpec if VAPI fields specify any. + // TODO: Remove limits: issues/56 + if res := vmClassSpec.Policies.Resources; !res.Requests.Memory.IsZero() || !res.Limits.Memory.IsZero() { + // TODO: Always override? + configSpec.MemoryAllocation = &types.ResourceAllocationInfo{ + Shares: &types.SharesInfo{ + Level: types.SharesLevelNormal, + }, + } + + if !res.Requests.Memory.IsZero() { + rsv := MemoryQuantityToMb(vmClassSpec.Policies.Resources.Requests.Memory) + configSpec.MemoryAllocation.Reservation = &rsv + } + if !res.Limits.Memory.IsZero() { + lim := MemoryQuantityToMb(vmClassSpec.Policies.Resources.Limits.Memory) + configSpec.MemoryAllocation.Limit = &lim + } + } + + return &configSpec +} + +// CreateConfigSpecForPlacement creates a ConfigSpec that is suitable for Placement. +// baseConfigSpec will likely be - or at least derived from - the ConfigSpec returned by CreateConfigSpec above. +func CreateConfigSpecForPlacement( + vmCtx context.VirtualMachineContextA2, + baseConfigSpec *types.VirtualMachineConfigSpec, + storageClassesToIDs map[string]string) *types.VirtualMachineConfigSpec { + + // TODO: If placement chokes on EthCards w/o a backing yet (NSX-T) remove those entries here. + deviceChangeCopy := make([]types.BaseVirtualDeviceConfigSpec, len(baseConfigSpec.DeviceChange)) + copy(deviceChangeCopy, baseConfigSpec.DeviceChange) + + configSpec := *baseConfigSpec + configSpec.DeviceChange = deviceChangeCopy + + // Add a dummy disk for placement: PlaceVmsXCluster expects there to always be at least one disk. + // Until we're in a position to have the OVF envelope here, add a dummy disk satisfy it. + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024 * 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + Profile: []types.BaseVirtualMachineProfileSpec{ + &types.VirtualMachineDefinedProfileSpec{ + ProfileId: storageClassesToIDs[vmCtx.VM.Spec.StorageClass], + }, + }, + }) + + if lib.IsInstanceStorageFSSEnabled() { + isVolumes := instancestorage.FilterVolumes(vmCtx.VM) + + for idx, dev := range CreateInstanceStorageDiskDevices(isVolumes) { + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate, + Device: dev, + Profile: []types.BaseVirtualMachineProfileSpec{ + &types.VirtualMachineDefinedProfileSpec{ + ProfileId: storageClassesToIDs[isVolumes[idx].PersistentVolumeClaim.InstanceVolumeClaim.StorageClass], + ProfileData: &types.VirtualMachineProfileRawData{ + ExtensionKey: "com.vmware.vim.sps", + }, + }, + }, + }) + } + } + + // TODO: Add more devices and fields + // - boot disks from OVA + // - storage profile/class + // - PVC volumes + // - Network devices (meh for now b/c of wcp constraints) + // - anything in ExtraConfig matter here? + // - any way to do the cluster modules for anti-affinity? + // - whatever else I'm forgetting + + return &configSpec +} + +// ConfigSpecFromVMClassDevices creates a ConfigSpec that adds the standalone hardware devices from +// the VMClass if any. This ConfigSpec will be used as the class ConfigSpec to CreateConfigSpec, with +// the rest of the class fields - like CPU count - applied on top. +func ConfigSpecFromVMClassDevices(vmClassSpec *vmopv1.VirtualMachineClassSpec) *types.VirtualMachineConfigSpec { + devsFromClass := CreatePCIDevicesFromVMClass(vmClassSpec.Hardware.Devices) + if len(devsFromClass) == 0 { + return nil + } + + configSpec := &types.VirtualMachineConfigSpec{} + for _, dev := range devsFromClass { + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: dev, + }) + } + return configSpec +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go new file mode 100644 index 000000000..0dd01e72c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go @@ -0,0 +1,278 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + vimtypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("CreateConfigSpec", func() { + const vmName = "dummy-vm" + + var ( + vmCtx context.VirtualMachineContextA2 + vmClassSpec *vmopv1.VirtualMachineClassSpec + vmImageStatus *vmopv1.VirtualMachineImageStatus + minCPUFreq uint64 + configSpec *vimtypes.VirtualMachineConfigSpec + classConfigSpec *vimtypes.VirtualMachineConfigSpec + err error + ) + + BeforeEach(func() { + vmClass := builder.DummyVirtualMachineClassA2() + vmClassSpec = &vmClass.Spec + vmImageStatus = &vmopv1.VirtualMachineImageStatus{Firmware: "efi"} + minCPUFreq = 2500 + + vm := builder.DummyVirtualMachineA2() + vm.Name = vmName + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithValues("vmName", vm.GetName()), + VM: vm, + } + }) + + It("Basic ConfigSpec assertions", func() { + configSpec = virtualmachine.CreateConfigSpec( + vmCtx, + nil, + vmClassSpec, + vmImageStatus, + minCPUFreq) + + Expect(configSpec).ToNot(BeNil()) + Expect(err).To(BeNil()) + Expect(configSpec.Name).To(Equal(vmName)) + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.NumCPUs).To(BeEquivalentTo(vmClassSpec.Hardware.Cpus)) + Expect(configSpec.MemoryMB).To(BeEquivalentTo(4 * 1024)) + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.Firmware).To(Equal(vmImageStatus.Firmware)) + }) + + Context("Use VM Class ConfigSpec", func() { + BeforeEach(func() { + classConfigSpec = &vimtypes.VirtualMachineConfigSpec{ + Name: "dont-use-this-dummy-VM", + Annotation: "test-annotation", + DeviceChange: []vimtypes.BaseVirtualDeviceConfigSpec{ + &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimtypes.VirtualE1000{ + VirtualEthernetCard: vimtypes.VirtualEthernetCard{ + VirtualDevice: vimtypes.VirtualDevice{ + Key: 4000, + }, + }, + }, + }, + }, + } + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.CreateConfigSpec( + vmCtx, + classConfigSpec, + vmClassSpec, + vmImageStatus, + minCPUFreq) + Expect(configSpec).ToNot(BeNil()) + }) + + It("Returns expected config spec", func() { + Expect(configSpec.Name).To(Equal(vmName)) + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.Annotation).To(Equal("test-annotation")) + Expect(configSpec.NumCPUs).To(BeEquivalentTo(vmClassSpec.Hardware.Cpus)) + Expect(configSpec.MemoryMB).To(BeEquivalentTo(4 * 1024)) + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.Firmware).To(Equal(vmImageStatus.Firmware)) + Expect(configSpec.DeviceChange).To(HaveLen(1)) + dSpec := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + _, ok := dSpec.Device.(*vimtypes.VirtualE1000) + Expect(ok).To(BeTrue()) + }) + }) +}) + +var _ = Describe("CreateConfigSpecForPlacement", func() { + + var ( + vmCtx context.VirtualMachineContextA2 + storageClassesToIDs map[string]string + baseConfigSpec *vimtypes.VirtualMachineConfigSpec + configSpec *vimtypes.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + baseConfigSpec = &vimtypes.VirtualMachineConfigSpec{} + storageClassesToIDs = map[string]string{} + + vm := builder.DummyVirtualMachineA2() + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithValues("vmName", vm.GetName()), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.CreateConfigSpecForPlacement( + vmCtx, + baseConfigSpec, + storageClassesToIDs) + Expect(configSpec).ToNot(BeNil()) + }) + + Context("Returns expected ConfigSpec", func() { + BeforeEach(func() { + baseConfigSpec = &vimtypes.VirtualMachineConfigSpec{ + Name: "dummy-VM", + Annotation: "test-annotation", + NumCPUs: 42, + MemoryMB: 4096, + Firmware: "secret-sauce", + DeviceChange: []vimtypes.BaseVirtualDeviceConfigSpec{ + &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimtypes.VirtualPCIPassthrough{ + VirtualDevice: vimtypes.VirtualDevice{ + Backing: &vimtypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + }, + } + }) + + It("Placement ConfigSpec contains expected field set sans ethernet device from class config spec", func() { + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.Annotation).To(Equal(baseConfigSpec.Annotation)) + Expect(configSpec.NumCPUs).To(Equal(baseConfigSpec.NumCPUs)) + Expect(configSpec.MemoryMB).To(Equal(baseConfigSpec.MemoryMB)) + Expect(configSpec.CpuAllocation).To(Equal(baseConfigSpec.CpuAllocation)) + Expect(configSpec.MemoryAllocation).To(Equal(baseConfigSpec.MemoryAllocation)) + Expect(configSpec.Firmware).To(Equal(baseConfigSpec.Firmware)) + + Expect(configSpec.DeviceChange).To(HaveLen(2)) + dSpec := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + _, ok := dSpec.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok).To(BeTrue()) + dSpec1 := configSpec.DeviceChange[1].GetVirtualDeviceConfigSpec() + _, ok = dSpec1.Device.(*vimtypes.VirtualDisk) + Expect(ok).To(BeTrue()) + }) + }) + + Context("When InstanceStorage is configured", func() { + const storagePolicyID = "storage-id-42" + var oldIsInstanceStorageFSSEnabled func() bool + + BeforeEach(func() { + oldIsInstanceStorageFSSEnabled = lib.IsInstanceStorageFSSEnabled + lib.IsInstanceStorageFSSEnabled = func() bool { return true } + + builder.AddDummyInstanceStorageVolumeA2(vmCtx.VM) + storageClassesToIDs[builder.DummyStorageClassName] = storagePolicyID + }) + + AfterEach(func() { + lib.IsInstanceStorageFSSEnabled = oldIsInstanceStorageFSSEnabled + }) + + It("ConfigSpec contains expected InstanceStorage devices", func() { + Expect(configSpec.DeviceChange).To(HaveLen(3)) + assertInstanceStorageDeviceChange(configSpec.DeviceChange[1], 256, storagePolicyID) + assertInstanceStorageDeviceChange(configSpec.DeviceChange[2], 512, storagePolicyID) + }) + }) +}) + +var _ = Describe("ConfigSpecFromVMClassDevices", func() { + + var ( + vmClassSpec *vmopv1.VirtualMachineClassSpec + configSpec *vimtypes.VirtualMachineConfigSpec + ) + + Context("when Class specifies GPU/DDPIO in Hardware", func() { + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + + vmClassSpec.Hardware.Devices.VGPUDevices = []vmopv1.VGPUDevice{{ + ProfileName: "createplacementspec-profile", + }} + + vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices = []vmopv1.DynamicDirectPathIODevice{{ + VendorID: 20, + DeviceID: 30, + CustomLabel: "createplacementspec-label", + }} + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.ConfigSpecFromVMClassDevices(vmClassSpec) + }) + + It("Returns expected ConfigSpec", func() { + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.DeviceChange).To(HaveLen(2)) // One each for GPU an DDPIO above + + dSpec1 := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + dev1, ok := dSpec1.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok).To(BeTrue()) + pciDev1 := dev1.GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*vimtypes.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).To(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal(vmClassSpec.Hardware.Devices.VGPUDevices[0].ProfileName)) + + dSpec2 := configSpec.DeviceChange[1].GetVirtualDeviceConfigSpec() + dev2, ok2 := dSpec2.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok2).To(BeTrue()) + pciDev2 := dev2.GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*vimtypes.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).To(BeTrue()) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(BeEquivalentTo(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].DeviceID)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(BeEquivalentTo(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].VendorID)) + Expect(pciBacking2.CustomLabel).To(Equal(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].CustomLabel)) + }) + }) +}) + +func assertInstanceStorageDeviceChange( + deviceChange vimtypes.BaseVirtualDeviceConfigSpec, + expectedSizeGB int, + expectedStoragePolicyID string) { + + dc := deviceChange.GetVirtualDeviceConfigSpec() + Expect(dc.Operation).To(Equal(vimtypes.VirtualDeviceConfigSpecOperationAdd)) + Expect(dc.FileOperation).To(Equal(vimtypes.VirtualDeviceConfigSpecFileOperationCreate)) + + dev, ok := dc.Device.(*vimtypes.VirtualDisk) + Expect(ok).To(BeTrue()) + Expect(dev.CapacityInBytes).To(BeEquivalentTo(expectedSizeGB * 1024 * 1024 * 1024)) + + Expect(dc.Profile).To(HaveLen(1)) + profile, ok := dc.Profile[0].(*vimtypes.VirtualMachineDefinedProfileSpec) + Expect(ok).To(BeTrue()) + Expect(profile.ProfileId).To(Equal(expectedStoragePolicyID)) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go new file mode 100644 index 000000000..06325c5e6 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go @@ -0,0 +1,18 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "math" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func MemoryQuantityToMb(q resource.Quantity) int64 { + return int64(math.Ceil(float64(q.Value()) / float64(1024*1024))) +} + +func CPUQuantityToMhz(q resource.Quantity, cpuFreqMhz uint64) int64 { + return int64(math.Ceil(float64(q.MilliValue()) * float64(cpuFreqMhz) / float64(1000))) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go new file mode 100644 index 000000000..cb53717eb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var _ = Describe("CPUQuantityToMhz", func() { + + Context("Convert CPU units from milli-cores to MHz", func() { + It("return whole number for non-integer CPU quantity", func() { + q, err := resource.ParseQuantity("500m") + Expect(err).NotTo(HaveOccurred()) + freq := virtualmachine.CPUQuantityToMhz(q, 3225) + expectVal := int64(1613) + Expect(freq).Should(BeNumerically("==", expectVal)) + }) + + It("return whole number for integer CPU quantity", func() { + q, err := resource.ParseQuantity("1000m") + Expect(err).NotTo(HaveOccurred()) + freq := virtualmachine.CPUQuantityToMhz(q, 3225) + expectVal := int64(3225) + Expect(freq).Should(BeNumerically("==", expectVal)) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go new file mode 100644 index 000000000..7641eaacb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" +) + +func DeleteVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine) error { + + if _, err := vmutil.SetAndWaitOnPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + vcVM.Client(), + vmutil.ManagedObjectFromObject(vcVM), + false, + types.VirtualMachinePowerStatePoweredOff, + vmutil.ParsePowerOpMode(string(vmCtx.VM.Spec.PowerOffMode))); err != nil { + + return err + } + + t, err := vcVM.Destroy(vmCtx) + if err != nil { + return err + } + + if taskInfo, err := t.WaitForResult(vmCtx); err != nil { + if taskInfo != nil { + vmCtx.Logger.V(5).Error(err, "destroy VM task failed", "taskInfo", taskInfo) + } + return errors.Wrapf(err, "destroy VM task failed") + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go new file mode 100644 index 000000000..3be4c39f5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func deleteTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), + VM: builder.DummyVirtualMachineA2(), + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Deletes VM that is off", func() { + moID := vcVM.Reference().Value + Expect(ctx.GetVMFromMoID(moID)).ToNot(BeNil()) + + t, err := vcVM.PowerOff(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Wait(ctx)).To(Succeed()) + + err = virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) + Expect(err).ToNot(HaveOccurred()) + + Expect(ctx.GetVMFromMoID(moID)).To(BeNil()) + }) + + It("Deletes VM that is on", func() { + moID := vcVM.Reference().Value + Expect(ctx.GetVMFromMoID(moID)).ToNot(BeNil()) + + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + err = virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) + Expect(err).ToNot(HaveOccurred()) + + Expect(ctx.GetVMFromMoID(moID)).To(BeNil()) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go new file mode 100644 index 000000000..1152ea435 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go @@ -0,0 +1,100 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + vimTypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +const ( + // A negative device range is traditionally used. + pciDevicesStartDeviceKey = int32(-200) + instanceStorageStartDeviceKey = int32(-300) +) + +func CreatePCIPassThroughDevice(deviceKey int32, backingInfo vimTypes.BaseVirtualDeviceBackingInfo) vimTypes.BaseVirtualDevice { + device := &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Key: deviceKey, + Backing: backingInfo, + }, + } + return device +} + +// CreatePCIDevicesFromConfigSpec creates vim25 VirtualDevices from the specified list of PCI devices from the VM Class ConfigSpec. +func CreatePCIDevicesFromConfigSpec(pciDevsFromConfigSpec []*vimTypes.VirtualPCIPassthrough) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(pciDevsFromConfigSpec)) + + deviceKey := pciDevicesStartDeviceKey + + for i := range pciDevsFromConfigSpec { + dev := pciDevsFromConfigSpec[i] + dev.Key = deviceKey + devices = append(devices, dev) + deviceKey-- + } + + return devices +} + +// CreatePCIDevicesFromVMClass creates vim25 VirtualDevices from the specified list of PCI devices from VM Class spec. +func CreatePCIDevicesFromVMClass(pciDevicesFromVMClass vmopv1.VirtualDevices) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(pciDevicesFromVMClass.VGPUDevices)+len(pciDevicesFromVMClass.DynamicDirectPathIODevices)) + + deviceKey := pciDevicesStartDeviceKey + + for _, vGPU := range pciDevicesFromVMClass.VGPUDevices { + backingInfo := &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: vGPU.ProfileName, + } + dev := CreatePCIPassThroughDevice(deviceKey, backingInfo) + devices = append(devices, dev) + deviceKey-- + } + + for _, dynamicDirectPath := range pciDevicesFromVMClass.DynamicDirectPathIODevices { + allowedDev := vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: int32(dynamicDirectPath.VendorID), + DeviceId: int32(dynamicDirectPath.DeviceID), + } + backingInfo := &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev}, + CustomLabel: dynamicDirectPath.CustomLabel, + } + dev := CreatePCIPassThroughDevice(deviceKey, backingInfo) + devices = append(devices, dev) + deviceKey-- + } + + return devices +} + +func CreateInstanceStorageDiskDevices(isVolumes []vmopv1.VirtualMachineVolume) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(isVolumes)) + deviceKey := instanceStorageStartDeviceKey + + for _, volume := range isVolumes { + device := &vimTypes.VirtualDisk{ + CapacityInBytes: volume.PersistentVolumeClaim.InstanceVolumeClaim.Size.Value(), + VirtualDevice: vimTypes.VirtualDevice{ + Key: deviceKey, + Backing: &vimTypes.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(false), + }, + }, + VDiskId: &vimTypes.ID{ + Id: constants.InstanceStorageVDiskID, + }, + } + devices = append(devices, device) + deviceKey-- + } + + return devices +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go new file mode 100644 index 000000000..424f8b527 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go @@ -0,0 +1,25 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +func GetGuestHeartBeatStatus( + ctx context.Context, + vm *object.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { + + var o mo.VirtualMachine + if err := vm.Properties(ctx, vm.Reference(), []string{"guestHeartbeatStatus"}, &o); err != nil { + return "", err + } + + return vmopv1.GuestHeartbeatStatus(o.GuestHeartbeatStatus), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go new file mode 100644 index 000000000..f1f13db2b --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go @@ -0,0 +1,64 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "fmt" + "net/http" + + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vapi/vcenter" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +const ( + sourceVirtualMachineType = "VirtualMachine" + + // vAPICtxActIDHttpHeader represents the http header in vAPI to pass down the activation ID. + vAPICtxActIDHttpHeader = "vapi-ctx-actid" + + itemDescriptionFormat = "virtualmachinepublishrequest.vmoperator.vmware.com: %s\n" +) + +func CreateOVF( + vmCtx context.VirtualMachineContextA2, + client *rest.Client, + vmPubReq *vmopv1.VirtualMachinePublishRequest, + cl *imgregv1a1.ContentLibrary, + actID string) (string, error) { + + // Use VM Operator specific description so that we can link published items + // to the vmPub if anything unexpected happened. + descriptionPrefix := fmt.Sprintf(itemDescriptionFormat, string(vmPubReq.UID)) + createSpec := vcenter.CreateSpec{ + Name: vmPubReq.Status.TargetRef.Item.Name, + Description: descriptionPrefix + vmPubReq.Status.TargetRef.Item.Description, + } + + source := vcenter.ResourceID{ + Type: sourceVirtualMachineType, + Value: vmCtx.VM.Status.UniqueID, + } + + target := vcenter.LibraryTarget{ + LibraryID: string(cl.Spec.UUID), + } + + ovf := vcenter.OVF{ + Spec: createSpec, + Source: source, + Target: target, + } + + vmCtx.Logger.Info("Creating OVF from VM", "spec", ovf, "actId", actID) + + // Use vmpublish uid as the act id passed down to the content library service, so that we can track + // the task status by the act id. + ctxHeader := client.WithHeader(vmCtx, http.Header{vAPICtxActIDHttpHeader: []string{actID}}) + return vcenter.NewManager(client).CreateOVF(ctxHeader, ovf) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go new file mode 100644 index 000000000..657f4de15 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func publishTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vm *vmopv1.VirtualMachine + cl *imgregv1a1.ContentLibrary + vmPub *vmopv1.VirtualMachinePublishRequest + vmCtx context.VirtualMachineContextA2 + vmPubCtx context.VirtualMachinePublishRequestContext + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true, WithContentLibrary: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + + vm = builder.DummyVirtualMachineA2() + vm.Status.UniqueID = vcVM.Reference().Value + cl = builder.DummyContentLibrary("dummy-cl", "dummy-ns", ctx.ContentLibraryID) + vmPub = builder.DummyVirtualMachinePublishRequestA2("dummy-vmpub", "dummy-ns", + vcVM.Name(), "dummy-item-name", "dummy-cl") + vmPub.Status.SourceRef = &vmPub.Spec.Source + vmPub.Status.TargetRef = &vmPub.Spec.Target + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), + VM: vm, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Publishes VM that is off", func() { + t, err := vcVM.PowerOff(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Wait(ctx)).To(Succeed()) + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) + + It("Publishes VM that is on", func() { + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) + + // TODO: update after vcsim bug is resolved. + // Currently if cl doesn't exist, vcsim set notFound http code + // but doesn't return immediately, which cause a panic error. + XIt("returns error if target content library does not exist", func() { + vmPubCtx.ContentLibrary.Spec.UUID = "12345" + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).To(HaveOccurred()) + Expect(itemID).To(BeEmpty()) + }) + + // TODO: vcsim currently doesn't check if an item already exists in the cl. + XIt("returns error if target content library item already exists", func() { + vmPubCtx.VMPublishRequest.Spec.Target.Item.Name = ctx.ContentLibraryImageName + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go new file mode 100644 index 000000000..8ba9c7376 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go @@ -0,0 +1,41 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/storage" +) + +// GetDefaultDiskProvisioningType gets the default disk provisioning type specified for the VM. +func GetDefaultDiskProvisioningType( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + storageProfileID string) (string, error) { + + switch vmCtx.VM.Spec.Advanced.DefaultVolumeProvisioningMode { + case vmopv1.VirtualMachineVolumeProvisioningModeThin: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil + case vmopv1.VirtualMachineVolumeProvisioningModeThick: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThick), nil + case vmopv1.VirtualMachineVolumeProvisioningModeThickEagerZero: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeEagerZeroedThick), nil + } + + if storageProfileID != "" { + provisioning, err := storage.GetDiskProvisioningForProfile(vmCtx, vcClient, storageProfileID) + if err != nil { + return "", err + } + if provisioning != "" { + return provisioning, nil + } + } + + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go new file mode 100644 index 000000000..12c5c35fd --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ClusterComputeResource", ccrTests) + Describe("Delete", deleteTests) + Describe("Publish", publishTests) +} + +var suite = builder.NewTestSuite() + +func TestClusterModules(t *testing.T) { + suite.Register(t, "vSphere Provider VirtualMachine Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go new file mode 100644 index 000000000..cc311b468 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go @@ -0,0 +1,65 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha512" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +func GetWebConsoleTicket( + vmCtx context.VirtualMachineContextA2, + vm *object.VirtualMachine, + pubKey string) (string, error) { + + vmCtx.Logger.V(5).Info("GetWebMKSTicket") + + ticket, err := vm.AcquireTicket(vmCtx, string(types.VirtualMachineTicketTypeWebmks)) + if err != nil { + return "", err + } + + url := fmt.Sprintf("wss://%s:%d/ticket/%s", ticket.Host, ticket.Port, ticket.Ticket) + return EncryptWebMKS(pubKey, url) +} + +func EncryptWebMKS(pubKey string, plaintext string) (string, error) { + block, _ := pem.Decode([]byte(pubKey)) + if block == nil || block.Type != "PUBLIC KEY" { + return "", errors.New("failed to decode PEM block containing public key") + } + pub, err := x509.ParsePKCS1PublicKey(block.Bytes) + if err != nil { + return "", err + } + cipherbytes, err := rsa.EncryptOAEP(sha512.New(), rand.Reader, pub, []byte(plaintext), nil) + if err != nil { + return "", err + } + ciphertext := base64.StdEncoding.EncodeToString(cipherbytes) + return ciphertext, nil +} + +func DecryptWebMKS(privKey *rsa.PrivateKey, ciphertext string) (string, error) { + decoded, err := base64.StdEncoding.DecodeString(ciphertext) + if err != nil { + return "", err + } + decrypted, err := rsa.DecryptOAEP(sha512.New(), rand.Reader, privKey, decoded, nil) + if err != nil { + return "", err + } + return string(decrypted), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go new file mode 100644 index 000000000..50a05cc72 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "crypto/rsa" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("Webconsole Ticket", func() { + + Context("EncryptWebMKS", func() { + var ( + privateKey *rsa.PrivateKey + publicKeyPem string + ) + + BeforeEach(func() { + privateKey, publicKeyPem = builder.WebConsoleRequestKeyPair() + }) + + It("Encrypts a string correctly", func() { + plaintext := "HelloWorld2" + ciphertext, err := virtualmachine.EncryptWebMKS(publicKeyPem, plaintext) + Expect(err).ShouldNot(HaveOccurred()) + decrypted, err := virtualmachine.DecryptWebMKS(privateKey, ciphertext) + Expect(err).ShouldNot(HaveOccurred()) + Expect(decrypted).To(Equal(plaintext)) + }) + + It("Error on invalid public key", func() { + plaintext := "HelloWorld3" + _, err := virtualmachine.EncryptWebMKS("invalid-pub-key", plaintext) + Expect(err).Should(HaveOccurred()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go new file mode 100644 index 000000000..1546399e8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go @@ -0,0 +1,265 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/task" + vimTypes "github.com/vmware/govmomi/vim25/types" + apiEquality "k8s.io/apimachinery/pkg/api/equality" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" +) + +const ( + // OvfEnvironmentTransportGuestInfo is the OVF transport type that uses + // GuestInfo. The other valid type is "iso". + OvfEnvironmentTransportGuestInfo = "com.vmware.guestInfo" +) + +type BootstrapData struct { + Data map[string]string + VAppData map[string]string + VAppExData map[string]map[string]string +} + +type TemplateRenderFunc func(string, string) string + +type BootstrapArgs struct { + BootstrapData + + TemplateRenderFn TemplateRenderFunc + NetworkResults network.NetworkInterfaceResults + Hostname string + DNSServers []string + SearchSuffixes []string +} + +func DoBootstrap( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + k8sClient ctrl.Client, + networkResults network.NetworkInterfaceResults, + bootstrapData BootstrapData) error { + + bootstrap := &vmCtx.VM.Spec.Bootstrap + cloudInit := bootstrap.CloudInit + linuxPrep := bootstrap.LinuxPrep + sysPrep := bootstrap.Sysprep + vAppConfig := bootstrap.VAppConfig + + bootstrapArgs, err := getBootstrapArgs(vmCtx, k8sClient, cloudInit != nil, networkResults, bootstrapData) + if err != nil { + return err + } + + if vAppConfig != nil { + // I think the intention was to only apply this to vAppData. Old code would apply it to entire + // Data map but for like SysPrep that data may be base64/gzip'd, and we'd do the template stuff + // prior to plain texting it. + bootstrapArgs.TemplateRenderFn = GetTemplateRenderFunc(vmCtx, bootstrapArgs) + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + var customSpec *vimTypes.CustomizationSpec + + switch { + case cloudInit != nil: + configSpec, customSpec, err = BootStrapCloudInit(vmCtx, config, cloudInit, bootstrapArgs) + case linuxPrep != nil: + configSpec, customSpec, err = BootStrapLinuxPrep(vmCtx, config, linuxPrep, vAppConfig, bootstrapArgs) + case sysPrep != nil: + configSpec, customSpec, err = BootstrapSysPrep(vmCtx, config, sysPrep, vAppConfig, bootstrapArgs) + case vAppConfig != nil: + configSpec, customSpec, err = BootstrapVAppConfig(vmCtx, config, vAppConfig, bootstrapArgs) + default: + // Old code fell back to LinuxPrep. Is that really appropriate anymore? + linuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{HardwareClockIsUTC: true} + configSpec, customSpec, err = BootStrapLinuxPrep(vmCtx, config, linuxPrep, nil, bootstrapArgs) + } + + if err != nil { + return fmt.Errorf("failed to create bootstrap data: %w", err) + } + + if configSpec != nil { + err := doReconfigure(vmCtx, vcVM, configSpec) + if err != nil { + return fmt.Errorf("boostrap reconfigure failed: %w", err) + } + } + + if customSpec != nil { + err := doCustomize(vmCtx, vcVM, config, customSpec) + if err != nil { + return fmt.Errorf("boostrap customize failed: %w", err) + } + } + + return nil +} + +func getBootstrapArgs( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrl.Client, + isCloudInit bool, + networkResults network.NetworkInterfaceResults, + bootstrapData BootstrapData) (*BootstrapArgs, error) { + + bootstrapArgs := BootstrapArgs{ + BootstrapData: bootstrapData, + NetworkResults: networkResults, + Hostname: vmCtx.VM.Spec.Network.HostName, + } + + if bootstrapArgs.Hostname == "" { + bootstrapArgs.Hostname = vmCtx.VM.Name + } + + // If the VM is missing DNS info - that is, it did not specify DNS for the interfaces - populate that + // now from the SV global configuration. Note that the VM is probably OK as long as at least one + // interface has DNS info, but we would previously set it for every interface so keep doing that + // here. Similarly, we didn't populate SearchDomains for non-TKG VMs so we don't here either. This is + // all a little nuts & complicated and probably not correct for every situation. + isTKG := hasTKGLabels(vmCtx.VM.Labels) + missingDNSInfo := false + for _, r := range networkResults.Results { + if r.DHCP4 || r.DHCP6 { + continue + } + + if len(r.Nameservers) == 0 || (isTKG && len(r.SearchDomains) == 0) { + missingDNSInfo = true + break + } + } + + if missingDNSInfo { + nameservers, searchSuffixes, err := config.GetDNSInformationFromConfigMap(k8sClient) + if err != nil && ctrl.IgnoreNotFound(err) != nil { + // This ConfigMap doesn't exist in certain test envs. + return nil, err + } + + // GOSC will use these for its global config. + bootstrapArgs.DNSServers = nameservers + bootstrapArgs.SearchSuffixes = searchSuffixes + + if isCloudInit { + // Previously we would apply the global DNS config to every interface so do that here too. + for i := range networkResults.Results { + r := &networkResults.Results[i] + + if r.DHCP4 || r.DHCP6 { + continue + } + + if len(r.Nameservers) == 0 { + r.Nameservers = nameservers + } + if isTKG && len(r.SearchDomains) == 0 { + r.SearchDomains = searchSuffixes + } + } + } + } + + return &bootstrapArgs, nil +} + +func hasTKGLabels(vmLabels map[string]string) bool { + const ( + // CAPWClusterRoleLabelKey is the key for the label applied to a VM that was + // created by CAPW. + CAPWClusterRoleLabelKey = "capw.vmware.com/cluster.role" //nolint:gosec + + // CAPVClusterRoleLabelKey is the key for the label applied to a VM that was + // created by CAPV. + CAPVClusterRoleLabelKey = "capv.vmware.com/cluster.role" + ) + + _, ok := vmLabels[CAPWClusterRoleLabelKey] + if !ok { + _, ok = vmLabels[CAPVClusterRoleLabelKey] + } + return ok +} + +func doReconfigure( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + configSpec *vimTypes.VirtualMachineConfigSpec) error { + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("Customization Reconfigure", "configSpec", configSpec) + + if err := resources.NewVMFromObject(vcVM).Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "customization reconfigure failed") + return err + } + } + + return nil +} + +func doCustomize( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + customSpec *vimTypes.CustomizationSpec) error { + + if vmCtx.VM.Annotations[constants.VSphereCustomizationBypassKey] == constants.VSphereCustomizationBypassDisable { + vmCtx.Logger.Info("Skipping vsphere customization because of vsphere-customization bypass annotation") + return nil + } + + if IsCustomizationPendingExtraConfig(config.ExtraConfig) { + vmCtx.Logger.Info("Skipping customization because it is already pending") + // TODO: We should really determine if the pending customization is stale, clear it + // if so, and then re-customize. Otherwise, the Customize call could perpetually fail + // preventing power on. + return nil + } + + vmCtx.Logger.Info("Customizing VM", "customizationSpec", *customSpec) + if err := resources.NewVMFromObject(vcVM).Customize(vmCtx, *customSpec); err != nil { + // isCustomizationPendingExtraConfig() above is supposed to prevent this error, but + // handle it explicitly here just in case so VM reconciliation can proceed. + if !isCustomizationPendingError(err) { + return err + } + } + + return nil +} + +func IsCustomizationPendingExtraConfig(extraConfig []vimTypes.BaseOptionValue) bool { + for _, opt := range extraConfig { + if optValue := opt.GetOptionValue(); optValue != nil { + if optValue.Key == constants.GOSCPendingExtraConfigKey { + return optValue.Value.(string) != "" + } + } + } + return false +} + +func isCustomizationPendingError(err error) bool { + if te, ok := err.(task.Error); ok { + if _, ok := te.Fault().(*vimTypes.CustomizationPending); ok { + return true + } + } + return false +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go new file mode 100644 index 000000000..9e46536d4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go @@ -0,0 +1,186 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + "strings" + + "github.com/vmware/govmomi/vim25/types" + "gopkg.in/yaml.v2" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +type CloudInitMetadata struct { + InstanceID string `yaml:"instance-id,omitempty"` + LocalHostname string `yaml:"local-hostname,omitempty"` + Hostname string `yaml:"hostname,omitempty"` + Network network.Netplan `yaml:"network,omitempty"` + PublicKeys string `yaml:"public-keys,omitempty"` +} + +func BootStrapCloudInit( + vmCtx context.VirtualMachineContextA2, + config *types.VirtualMachineConfigInfo, + cloudInitSpec *vmopv1.VirtualMachineBootstrapCloudInitSpec, + bsArgs *BootstrapArgs) (*types.VirtualMachineConfigSpec, *types.CustomizationSpec, error) { + + netPlan, err := network.NetPlanCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create NetPlan customization: %w", err) + } + + sshPublicKeys := bsArgs.BootstrapData.Data["ssh-public-keys"] + if len(cloudInitSpec.SSHAuthorizedKeys) > 0 { + sshPublicKeys = strings.Join(cloudInitSpec.SSHAuthorizedKeys, "\n") + } + + metadata, err := GetCloudInitMetadata(string(vmCtx.VM.UID), bsArgs.Hostname, netPlan, sshPublicKeys) + if err != nil { + return nil, nil, err + } + + var userdata string + if cloudInitSpec.RawCloudConfig.Name != "" { + // Check for the 'user-data' key as per official contract and API documentation. + // Additionally, to support the cluster bootstrap data supplied by CAPBK's secret, + // we check for a 'value' key when 'user-data' is not supplied. + // The 'value' key lookup will eventually be deprecated. + for _, k := range []string{cloudInitSpec.RawCloudConfig.Key, "user-data", "value"} { + if k != "" { + userdata = bsArgs.BootstrapData.Data[k] + if userdata != "" { + break + } + } + } + + // NOTE: The old code didn't error out if userdata wasn't found, so keep going. + + } else { + return nil, nil, fmt.Errorf("TODO: inlined CloudConfig") + } + + var configSpec *types.VirtualMachineConfigSpec + var customSpec *types.CustomizationSpec + + switch vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] { + case constants.CloudInitTypeValueCloudInitPrep: + configSpec, customSpec, err = GetCloudInitPrepCustSpec(metadata, userdata) + case constants.CloudInitTypeValueGuestInfo, "": + fallthrough + default: + configSpec, err = GetCloudInitGuestInfoCustSpec(config, metadata, userdata) + } + + if err != nil { + return nil, nil, err + } + + return configSpec, customSpec, nil +} + +func GetCloudInitMetadata( + uid string, + hostname string, + netplan *network.Netplan, + sshPublicKeys string) (string, error) { + + metadata := &CloudInitMetadata{ + InstanceID: uid, + LocalHostname: hostname, + Hostname: hostname, + Network: *netplan, + PublicKeys: sshPublicKeys, + } + + metadataBytes, err := yaml.Marshal(metadata) + if err != nil { + return "", fmt.Errorf("yaml marshalling of cloud-init metadata failed: %w", err) + } + + return string(metadataBytes), nil +} + +func GetCloudInitPrepCustSpec( + metadata, userdata string) (*types.VirtualMachineConfigSpec, *types.CustomizationSpec, error) { + + if userdata != "" { + // Ensure the data is normalized first to plain-text. + plainText, err := util.TryToDecodeBase64Gzip([]byte(userdata)) + if err != nil { + return nil, nil, fmt.Errorf("decoding cloud-init prep userdata failed: %w", err) + } + + userdata = plainText + } + + // FIXME: This is the old behavior but isn't quite correct: we need current config, and only + // do this if the transport isn't already set. Otherwise, this always results in a Reconfigure. + configSpec := &types.VirtualMachineConfigSpec{ + VAppConfig: &types.VmConfigSpec{ + // Ensure the transport is guestInfo in case the VM does not have + // a CD-ROM device required to use the ISO transport. + OvfEnvironmentTransport: []string{OvfEnvironmentTransportGuestInfo}, + }, + } + + customSpec := &types.CustomizationSpec{ + Identity: &internal.CustomizationCloudinitPrep{ + Metadata: metadata, + Userdata: userdata, + }, + } + + return configSpec, customSpec, nil +} + +func GetCloudInitGuestInfoCustSpec( + config *types.VirtualMachineConfigInfo, + metadata, userdata string) (*types.VirtualMachineConfigSpec, error) { + + encodedMetadata, err := util.EncodeGzipBase64(metadata) + if err != nil { + return nil, fmt.Errorf("encoding cloud-init metadata failed: %w", err) + } + + extraConfig := map[string]string{ + constants.CloudInitGuestInfoMetadata: encodedMetadata, + constants.CloudInitGuestInfoMetadataEncoding: "gzip+base64", + } + + if userdata != "" { + // Ensure the data is normalized first to plain-text. + plainText, err := util.TryToDecodeBase64Gzip([]byte(userdata)) + if err != nil { + return nil, fmt.Errorf("decoding cloud-init userdata failed: %w", err) + } + + encodedUserdata, err := util.EncodeGzipBase64(plainText) + if err != nil { + return nil, fmt.Errorf("encoding cloud-init userdata failed: %w", err) + } + + extraConfig[constants.CloudInitGuestInfoUserdata] = encodedUserdata + extraConfig[constants.CloudInitGuestInfoUserdataEncoding] = "gzip+base64" + } + + // FIXME: This is the old behavior but isn't quite correct: we really need the current ExtraConfig, + // and then only add/update it if new or updated (so we don't customize with stale data). Also, + // always setting VAppConfigRemoved isn't correct: should only set it if the VM has vApp data. + // As-is we will always Reconfigure the VM. + configSpec := &types.VirtualMachineConfigSpec{} + configSpec.ExtraConfig = util.AppendNewExtraConfigValues(config.ExtraConfig, extraConfig) + // Remove the VAppConfig to ensure Cloud-Init inside the guest does not + // activate and prefer the OVF datasource over the VMware datasource. + configSpec.VAppConfigRemoved = types.NewBool(true) // FIXME + + return configSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go new file mode 100644 index 000000000..7badf04c8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go @@ -0,0 +1,398 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + "encoding/base64" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + "gopkg.in/yaml.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("CloudInit Bootstrap", func() { + const ( + cloudInitMetadata = "cloud-init-metadata" + cloudInitUserdata = "cloud-init-userdata" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + + metaData string + userData string + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + + // Set defaults. + metaData = cloudInitMetadata + userData = cloudInitUserdata + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + extraConfigToMap := func(input []types.BaseOptionValue) map[string]string { + output := make(map[string]string) + for _, opt := range input { + if optValue := opt.GetOptionValue(); optValue != nil { + // Only set string type values + if val, ok := optValue.Value.(string); ok { + output[optValue.Key] = val + } + } + } + return output + } + + // v1a1 tests really only tested the lower level functions individually. Those tests are ported after + // this Context, but we should focus more on testing via this just method. + Context("BootStrapCloudInit", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + cloudInitSpec *vmopv1.VirtualMachineBootstrapCloudInitSpec + ) + + BeforeEach(func() { + cloudInitSpec = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-init-bootstrap-test", + Namespace: "test-ns", + UID: "my-vm-uuid", + Annotations: map[string]string{}, + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootStrapCloudInit( + vmCtx, + configInfo, + cloudInitSpec, + &bsArgs, + ) + }) + + Context("CloudInit Inlined Config", func() { + It("Returns TODO", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("TODO")) + }) + }) + + Context("CloudInit Data Config", func() { + BeforeEach(func() { + cloudInitSpec.RawCloudConfig.Name = "my-data" + cloudInitSpec.RawCloudConfig.Key = "my-key" + bsArgs.Data[cloudInitSpec.RawCloudConfig.Key] = cloudInitUserdata + }) + + Context("Via CloudInitPrep", func() { + BeforeEach(func() { + vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] = constants.CloudInitTypeValueCloudInitPrep + }) + + It("Returns success", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec()).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport).To(HaveLen(1)) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).ToNot(BeEmpty()) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("Via GuestInfo", func() { + BeforeEach(func() { + vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] = constants.CloudInitTypeValueGuestInfo + }) + + It("Returns Success", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfigRemoved).ToNot(BeNil()) + Expect(*configSpec.VAppConfigRemoved).To(BeTrue()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + + Expect(custSpec).To(BeNil()) + }) + + Context("Via CAPBK userdata in 'value' key", func() { + const otherUserData = cloudInitUserdata + "CAPBK" + + BeforeEach(func() { + bsArgs.Data[cloudInitSpec.RawCloudConfig.Key] = "" + cloudInitSpec.RawCloudConfig.Key = "" + bsArgs.Data["value"] = otherUserData + }) + + It("Returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoUserdata)) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + + data, err := util.TryToDecodeBase64Gzip([]byte(extraConfig[constants.CloudInitGuestInfoUserdata])) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(Equal(otherUserData)) + }) + }) + }) + }) + }) + + Context("GetCloudInitMetadata", func() { + var ( + uid string + hostName string + netPlan *network.Netplan + sshPublicKeys string + + mdYaml string + err error + ) + + BeforeEach(func() { + uid = "my-uid" + hostName = "my-hostname" + netPlan = &network.Netplan{ + Version: 42, + Ethernets: map[string]network.NetplanEthernet{ + "eth0": { + SetName: "eth0", + }, + }, + } + sshPublicKeys = "my-ssh-key" + }) + + JustBeforeEach(func() { + mdYaml, err = vmlifecycle.GetCloudInitMetadata(uid, hostName, netPlan, sshPublicKeys) + }) + + It("DoIt", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(mdYaml).ToNot(BeEmpty()) + + ciMetadata := &vmlifecycle.CloudInitMetadata{} + Expect(yaml.Unmarshal([]byte(mdYaml), ciMetadata)).To(Succeed()) + + Expect(ciMetadata.InstanceID).To(Equal(uid)) + Expect(ciMetadata.Hostname).To(Equal(hostName)) + Expect(ciMetadata.PublicKeys).To(Equal(sshPublicKeys)) + Expect(ciMetadata.Network.Version).To(Equal(42)) + Expect(ciMetadata.Network.Ethernets).To(HaveKey("eth0")) + }) + }) + + Context("GetCloudInitGuestInfoCustSpec", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + err error + ) + + JustBeforeEach(func() { + configSpec, err = vmlifecycle.GetCloudInitGuestInfoCustSpec(configInfo, metaData, userData) + }) + + Context("VAppConfig Disabled", func() { + It("Should disable the VAppConfig", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfigRemoved).ToNot(BeNil()) + Expect(*configSpec.VAppConfigRemoved).To(BeTrue()) + }) + }) + + Context("No userdata", func() { + BeforeEach(func() { + userData = "" + }) + + It("ConfigSpec.ExtraConfig to only have metadata", func() { + Expect(configSpec).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(2)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With userdata", func() { + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + userData = base64.StdEncoding.EncodeToString([]byte(cloudInitUserdata)) + }) + + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With gzipped, base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + data, err := util.EncodeGzipBase64(cloudInitUserdata) + Expect(err).ToNot(HaveOccurred()) + userData = data + }) + + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + }) + + Context("GetCloudInitPrepCustSpec", func() { + var ( + custSpec *types.CustomizationSpec + ) + + JustBeforeEach(func() { + var configSpec *types.VirtualMachineConfigSpec + var err error + + configSpec, custSpec, err = vmlifecycle.GetCloudInitPrepCustSpec(metaData, userData) + Expect(err).ToNot(HaveOccurred()) + + // Validate that Cloud-Init Prep always uses the GuestInfo transport for the CI Prep's meta and user data. + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec()).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport).To(HaveLen(1)) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + }) + + Context("No userdata", func() { + BeforeEach(func() { + userData = "" + }) + + It("Cust spec to only have metadata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(BeEmpty()) + }) + }) + + Context("With userdata", func() { + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("With base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + userData = base64.StdEncoding.EncodeToString([]byte(cloudInitUserdata)) + }) + + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("With gzipped, base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + data, err := util.EncodeGzipBase64(cloudInitUserdata) + Expect(err).ToNot(HaveOccurred()) + userData = data + }) + + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go new file mode 100644 index 000000000..0f954fd0f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go @@ -0,0 +1,55 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +func BootStrapLinuxPrep( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + linuxPrepSpec *vmopv1.VirtualMachineBootstrapLinuxPrepSpec, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + nicSettingMap, err := network.GuestOSCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create GOSC NIC mappings: %w", err) + } + + customSpec := &vimTypes.CustomizationSpec{ + Identity: &vimTypes.CustomizationLinuxPrep{ + HostName: &vimTypes.CustomizationFixedName{ + Name: bsArgs.Hostname, + }, + TimeZone: linuxPrepSpec.TimeZone, + HwClockUTC: vimTypes.NewBool(linuxPrepSpec.HardwareClockIsUTC), + }, + GlobalIPSettings: vimTypes.CustomizationGlobalIPSettings{ + DnsSuffixList: bsArgs.SearchSuffixes, + DnsServerList: bsArgs.DNSServers, + }, + NicSettingMap: nicSettingMap, + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + if vAppConfigSpec != nil { + configSpec = &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + } + + return configSpec, customSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go new file mode 100644 index 000000000..06fcd4a12 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go @@ -0,0 +1,156 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("LinuxPrep Bootstrap", func() { + const ( + macAddr = "43-AB-B4-1B-7E-87" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("BootStrapLinuxPrep", func() { + + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + linuxPrepSpec *vmopv1.VirtualMachineBootstrapLinuxPrepSpec + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + ) + + BeforeEach(func() { + linuxPrepSpec = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + vAppConfigSpec = nil + + bsArgs.Hostname = "my-hostname" + bsArgs.SearchSuffixes = []string{"suffix1", "suffix2"} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: "192.168.1.1", + IPCIDR: "192.168.1.10/24", + IsIPv4: true, + }, + }, + }, + } + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "linux-prep-bootstrap-test", + Namespace: "test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootStrapLinuxPrep( + vmCtx, + configInfo, + linuxPrepSpec, + vAppConfigSpec, + &bsArgs, + ) + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).To(BeNil()) + + Expect(custSpec).ToNot(BeNil()) + Expect(custSpec.GlobalIPSettings.DnsServerList).To(Equal(bsArgs.DNSServers)) + Expect(custSpec.GlobalIPSettings.DnsSuffixList).To(Equal(bsArgs.SearchSuffixes)) + + linuxSpec := custSpec.Identity.(*types.CustomizationLinuxPrep) + hostName := linuxSpec.HostName.(*types.CustomizationFixedName).Name + Expect(hostName).To(Equal(bsArgs.Hostname)) + Expect(linuxSpec.TimeZone).To(Equal(linuxPrepSpec.TimeZone)) + Expect(linuxSpec.HwClockUTC).ToNot(BeNil()) + Expect(*linuxSpec.HwClockUTC).To(Equal(linuxPrepSpec.HardwareClockIsUTC)) + + Expect(custSpec.NicSettingMap).To(HaveLen(len(bsArgs.NetworkResults.Results))) + Expect(custSpec.NicSettingMap[0].MacAddress).To(Equal(macAddr)) + }) + + Context("when has vAppConfig", func() { + const key, value = "fooKey", "fooValue" + + BeforeEach(func() { + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(custSpec).ToNot(BeNil()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + vmCs := configSpec.VAppConfig.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go new file mode 100644 index 000000000..a8fe2baea --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go @@ -0,0 +1,79 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + + vimTypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/apimachinery/pkg/api/equality" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/sysprep" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +func BootstrapSysPrep( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + sysPrepSpec *vmopv1.VirtualMachineBootstrapSysprepSpec, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + var data string + + if equality.Semantic.DeepEqual(sysPrepSpec.Sysprep, sysprep.Sysprep{}) { + var err error + + key := "unattend" + if sysPrepSpec.RawSysprep.Key != "" { + key = sysPrepSpec.RawSysprep.Key + } + + data = bsArgs.BootstrapData.Data[key] + if data == "" { + return nil, nil, fmt.Errorf("no Sysprep XML data with key %q", key) + } + + // Ensure the data is normalized first to plain-text. + data, err = util.TryToDecodeBase64Gzip([]byte(data)) + if err != nil { + return nil, nil, fmt.Errorf("decoding Sysprep unattend XML failed: %w", err) + } + + } else { + return nil, nil, fmt.Errorf("TODO: inlined Sysprep") + } + + nicSettingMap, err := network.GuestOSCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create GSOC adapter mappings: %w", err) + } + + customSpec := &vimTypes.CustomizationSpec{ + Identity: &vimTypes.CustomizationSysprepText{ + Value: data, + }, + GlobalIPSettings: vimTypes.CustomizationGlobalIPSettings{ + DnsSuffixList: bsArgs.SearchSuffixes, + DnsServerList: bsArgs.DNSServers, + }, + NicSettingMap: nicSettingMap, + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + if vAppConfigSpec != nil { + configSpec = &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + } + + return configSpec, customSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go new file mode 100644 index 000000000..2a61f0ac7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("SysPrep Bootstrap", func() { + const ( + macAddr = "43-AB-B4-1B-7E-87" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("BootStrapSysPrep", func() { + const unattendXML = "dummy-unattend-xml" + + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + sysPrepSpec *vmopv1.VirtualMachineBootstrapSysprepSpec + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + ) + + BeforeEach(func() { + sysPrepSpec = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + vAppConfigSpec = nil + + bsArgs.Data["unattend"] = unattendXML + bsArgs.SearchSuffixes = []string{"suffix1", "suffix2"} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: "192.168.1.1", + IPCIDR: "192.168.1.10/24", + IsIPv4: true, + }, + }, + }, + } + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sys-prep-bootstrap-test", + Namespace: "test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootstrapSysPrep( + vmCtx, + configInfo, + sysPrepSpec, + vAppConfigSpec, + &bsArgs, + ) + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).To(BeNil()) + + Expect(custSpec).ToNot(BeNil()) + Expect(custSpec.GlobalIPSettings.DnsServerList).To(Equal(bsArgs.DNSServers)) + Expect(custSpec.GlobalIPSettings.DnsSuffixList).To(Equal(bsArgs.SearchSuffixes)) + + sysPrepText := custSpec.Identity.(*types.CustomizationSysprepText) + Expect(sysPrepText.Value).To(Equal(unattendXML)) + + Expect(custSpec.NicSettingMap).To(HaveLen(len(bsArgs.NetworkResults.Results))) + Expect(custSpec.NicSettingMap[0].MacAddress).To(Equal(macAddr)) + }) + + Context("when has vAppConfig", func() { + const key, value = "fooKey", "fooValue" + + BeforeEach(func() { + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(custSpec).ToNot(BeNil()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + vmCs := configSpec.VAppConfig.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go new file mode 100644 index 000000000..e80ee1042 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go @@ -0,0 +1,453 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "bytes" + "errors" + "fmt" + "net" + "strings" + "text/template" + + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +func GetTemplateRenderFunc( + vmCtx context.VirtualMachineContextA2, + bsArgs *BootstrapArgs, +) TemplateRenderFunc { + + // There is a lot of duplication here, especially since the "template" types are the same in v1a1 + // and v1a2. We've conflated a lot of things here making this all a little nuts. + + networkDevicesStatusV1A1 := toTemplateNetworkStatusV1A1(bsArgs) + networkStatusV1A1 := v1alpha1.NetworkStatus{ + Devices: networkDevicesStatusV1A1, + Nameservers: bsArgs.DNSServers, + } + + networkDevicesStatusV1A2 := toTemplateNetworkStatus(bsArgs) + networkStatusV1A2 := v1alpha2.NetworkStatus{ + Devices: networkDevicesStatusV1A2, + Nameservers: bsArgs.DNSServers, + } + + // Oh dear. The VM itself really should not have been included here. + v1a1VM := &v1alpha1.VirtualMachine{} + _ = v1a1VM.ConvertFrom(vmCtx.VM) + + templateData := struct { + V1alpha1 v1alpha1.VirtualMachineTemplate + V1alpha2 v1alpha2.VirtualMachineTemplate + }{ + V1alpha1: v1alpha1.VirtualMachineTemplate{ + Net: networkStatusV1A1, + VM: v1a1VM, + }, + V1alpha2: v1alpha2.VirtualMachineTemplate{ + Net: networkStatusV1A2, + VM: vmCtx.VM, + }, + } + + v1a1FuncMap := v1a1TemplateFunctions(networkStatusV1A1, networkDevicesStatusV1A1) + v1a2FuncMap := v1a2TemplateFunctions(networkStatusV1A2, networkDevicesStatusV1A2) + + // Include both but should probably leave out v1a2 if we can identify this was originally a v1a1 VM. + funcMap := template.FuncMap{} + for k, v := range v1a1FuncMap { + funcMap[k] = v + } + for k, v := range v1a2FuncMap { + funcMap[k] = v + } + + // Skip parsing when encountering escape character('\{',"\}") + normalizeStr := func(str string) string { + if strings.Contains(str, "\\{") || strings.Contains(str, "\\}") { + str = strings.ReplaceAll(str, "\\{", "{") + str = strings.ReplaceAll(str, "\\}", "}") + } + return str + } + + // TODO: Don't log, return errors instead. + renderTemplate := func(name, templateStr string) string { + templ, err := template.New(name).Funcs(funcMap).Parse(templateStr) + if err != nil { + vmCtx.Logger.Error(err, "failed to parse template", "templateStr", templateStr) + return normalizeStr(templateStr) + } + var doc bytes.Buffer + err = templ.Execute(&doc, &templateData) + if err != nil { + vmCtx.Logger.Error(err, "failed to execute template", "templateStr", templateStr) + return normalizeStr(templateStr) + } + return normalizeStr(doc.String()) + } + + return renderTemplate +} + +func v1a1TemplateFunctions( + networkStatusV1A1 v1alpha1.NetworkStatus, + networkDevicesStatusV1A1 []v1alpha1.NetworkDeviceStatus) map[string]any { + + // Get the first IP address from the first NIC. + v1alpha1FirstIP := func() (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A1[0].IPAddresses[0], nil + } + + // Get the first NIC's MAC address. + v1alpha1FirstNicMacAddr := func() (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A1[0].MacAddress, nil + } + + // Get the first IP address from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha1FirstIPFromNIC := func(index int) (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A1) { + return "", errors.New("index out of bound") + } + return networkDevicesStatusV1A1[index].IPAddresses[0], nil + } + + // Get all IP addresses from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha1IPsFromNIC := func(index int) ([]string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return []string{""}, errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A1) { + return []string{""}, errors.New("index out of bound") + } + return networkDevicesStatusV1A1[index].IPAddresses, nil + } + + // Format the first occurred count of nameservers with specific delimiter + // A negative count number would mean format all nameservers + v1alpha1FormatNameservers := func(count int, delimiter string) (string, error) { + var nameservers []string + if len(networkStatusV1A1.Nameservers) == 0 { + return "", errors.New("no available nameservers, check with VI admin") + } + if count < 0 || count >= len(networkStatusV1A1.Nameservers) { + nameservers = networkStatusV1A1.Nameservers + return strings.Join(nameservers, delimiter), nil + } + nameservers = networkStatusV1A1.Nameservers[:count] + return strings.Join(nameservers, delimiter), nil + } + + // Get subnet mask from a CIDR notation IP address and prefix length + // if IP address and prefix length not valid, throw an error and template string won't be parsed + v1alpha1SubnetMask := func(cidr string) (string, error) { + _, ipv4Net, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + netmask := fmt.Sprintf("%d.%d.%d.%d", ipv4Net.Mask[0], ipv4Net.Mask[1], ipv4Net.Mask[2], ipv4Net.Mask[3]) + return netmask, nil + } + + // Format an IP address with default netmask CIDR + // if IP not valid, throw an error and template string won't be parsed + v1alpha1IP := func(IP string) (string, error) { + if net.ParseIP(IP) == nil { + return "", errors.New("input IP address not valid") + } + defaultMask := net.ParseIP(IP).DefaultMask() + ones, _ := defaultMask.Size() + expectedCidrNotation := IP + "/" + fmt.Sprintf("%d", int32(ones)) + return expectedCidrNotation, nil + } + + // Format an IP address with network length(eg. /24) or decimal + // notation (eg. 255.255.255.0). Format an IP/CIDR with updated mask. + // An empty mask causes just the IP to be returned. + v1alpha1FormatIP := func(s string, mask string) (string, error) { + // Get the IP address for the input string. + ip, _, err := net.ParseCIDR(s) + if err != nil { + ip = net.ParseIP(s) + if ip == nil { + return "", fmt.Errorf("input IP address not valid") + } + } + // Store the IP as a string back into s. + s = ip.String() + + // If no mask was provided then return just the IP. + if mask == "" { + return s, nil + } + + // The provided mask is a network length. + if strings.HasPrefix(mask, "/") { + s += mask + if _, _, err := net.ParseCIDR(s); err != nil { + return "", err + } + return s, nil + } + + // The provided mask is subnet mask. + maskIP := net.ParseIP(mask) + if maskIP == nil { + return "", fmt.Errorf("mask is an invalid IP") + } + + maskIPBytes := maskIP.To4() + if len(maskIPBytes) == 0 { + maskIPBytes = maskIP.To16() + } + + ipNet := net.IPNet{ + IP: ip, + Mask: net.IPMask(maskIPBytes), + } + s = ipNet.String() + + // Validate the ipNet is an IP/CIDR + if _, _, err := net.ParseCIDR(s); err != nil { + return "", fmt.Errorf("invalid ip net: %s", s) + } + + return s, nil + } + + return template.FuncMap{ + constants.V1alpha1FirstIP: v1alpha1FirstIP, + constants.V1alpha1FirstNicMacAddr: v1alpha1FirstNicMacAddr, + constants.V1alpha1FirstIPFromNIC: v1alpha1FirstIPFromNIC, + constants.V1alpha1IPsFromNIC: v1alpha1IPsFromNIC, + constants.V1alpha1FormatNameservers: v1alpha1FormatNameservers, + // These are more util function that we've conflated version namespaces. + constants.V1alpha1SubnetMask: v1alpha1SubnetMask, + constants.V1alpha1IP: v1alpha1IP, + constants.V1alpha1FormatIP: v1alpha1FormatIP, + } +} + +func toTemplateNetworkStatus(bsArgs *BootstrapArgs) []v1alpha2.NetworkDeviceStatus { + networkDevicesStatus := make([]v1alpha2.NetworkDeviceStatus, 0, len(bsArgs.NetworkResults.Results)) + + for _, result := range bsArgs.NetworkResults.Results { + // When using Sysprep, the MAC address must be in the format of "-". + // CloudInit normalizes it again to ":" when adding it to the netplan. + macAddr := strings.ReplaceAll(result.MacAddress, ":", "-") + + status := v1alpha2.NetworkDeviceStatus{ + MacAddress: macAddr, + } + + for _, ipConfig := range result.IPConfigs { + // We mostly only did IPv4 before so keep that going. + if ipConfig.IsIPv4 { + if status.Gateway4 == "" { + status.Gateway4 = ipConfig.Gateway + } + + status.IPAddresses = append(status.IPAddresses, ipConfig.IPCIDR) + } + } + + networkDevicesStatus = append(networkDevicesStatus, status) + } + + return networkDevicesStatus +} + +// This is basically identical to v1a1TemplateFunctions. +func v1a2TemplateFunctions( + networkStatusV1A2 v1alpha2.NetworkStatus, + networkDevicesStatusV1A2 []v1alpha2.NetworkDeviceStatus) map[string]any { + + // Get the first IP address from the first NIC. + v1alpha2FirstIP := func() (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A2[0].IPAddresses[0], nil + } + + // Get the first NIC's MAC address. + v1alpha2FirstNicMacAddr := func() (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A2[0].MacAddress, nil + } + + // Get the first IP address from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha2FirstIPFromNIC := func(index int) (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A2) { + return "", errors.New("index out of bound") + } + return networkDevicesStatusV1A2[index].IPAddresses[0], nil + } + + // Get all IP addresses from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha2IPsFromNIC := func(index int) ([]string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return []string{""}, errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A2) { + return []string{""}, errors.New("index out of bound") + } + return networkDevicesStatusV1A2[index].IPAddresses, nil + } + + // Format the first occurred count of nameservers with specific delimiter + // A negative count number would mean format all nameservers + v1alpha2FormatNameservers := func(count int, delimiter string) (string, error) { + var nameservers []string + if len(networkStatusV1A2.Nameservers) == 0 { + return "", errors.New("no available nameservers, check with VI admin") + } + if count < 0 || count >= len(networkStatusV1A2.Nameservers) { + nameservers = networkStatusV1A2.Nameservers + return strings.Join(nameservers, delimiter), nil + } + nameservers = networkStatusV1A2.Nameservers[:count] + return strings.Join(nameservers, delimiter), nil + } + + // Get subnet mask from a CIDR notation IP address and prefix length + // if IP address and prefix length not valid, throw an error and template string won't be parsed + v1alpha2SubnetMask := func(cidr string) (string, error) { + _, ipv4Net, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + netmask := fmt.Sprintf("%d.%d.%d.%d", ipv4Net.Mask[0], ipv4Net.Mask[1], ipv4Net.Mask[2], ipv4Net.Mask[3]) + return netmask, nil + } + + // Format an IP address with default netmask CIDR + // if IP not valid, throw an error and template string won't be parsed + v1alpha2IP := func(IP string) (string, error) { + if net.ParseIP(IP) == nil { + return "", errors.New("input IP address not valid") + } + defaultMask := net.ParseIP(IP).DefaultMask() + ones, _ := defaultMask.Size() + expectedCidrNotation := IP + "/" + fmt.Sprintf("%d", int32(ones)) + return expectedCidrNotation, nil + } + + // Format an IP address with network length(eg. /24) or decimal + // notation (eg. 255.255.255.0). Format an IP/CIDR with updated mask. + // An empty mask causes just the IP to be returned. + v1alpha2FormatIP := func(s string, mask string) (string, error) { + // Get the IP address for the input string. + ip, _, err := net.ParseCIDR(s) + if err != nil { + ip = net.ParseIP(s) + if ip == nil { + return "", fmt.Errorf("input IP address not valid") + } + } + // Store the IP as a string back into s. + s = ip.String() + + // If no mask was provided then return just the IP. + if mask == "" { + return s, nil + } + + // The provided mask is a network length. + if strings.HasPrefix(mask, "/") { + s += mask + if _, _, err := net.ParseCIDR(s); err != nil { + return "", err + } + return s, nil + } + + // The provided mask is subnet mask. + maskIP := net.ParseIP(mask) + if maskIP == nil { + return "", fmt.Errorf("mask is an invalid IP") + } + + maskIPBytes := maskIP.To4() + if len(maskIPBytes) == 0 { + maskIPBytes = maskIP.To16() + } + + ipNet := net.IPNet{ + IP: ip, + Mask: net.IPMask(maskIPBytes), + } + s = ipNet.String() + + // Validate the ipNet is an IP/CIDR + if _, _, err := net.ParseCIDR(s); err != nil { + return "", fmt.Errorf("invalid ip net: %s", s) + } + + return s, nil + } + + return template.FuncMap{ + constants.V1alpha2FirstIP: v1alpha2FirstIP, + constants.V1alpha2FirstNicMacAddr: v1alpha2FirstNicMacAddr, + constants.V1alpha2FirstIPFromNIC: v1alpha2FirstIPFromNIC, + constants.V1alpha2IPsFromNIC: v1alpha2IPsFromNIC, + constants.V1alpha2FormatNameservers: v1alpha2FormatNameservers, + // These are more util function that we've conflated version namespaces. + constants.V1alpha2SubnetMask: v1alpha2SubnetMask, + constants.V1alpha2IP: v1alpha2IP, + constants.V1alpha2FormatIP: v1alpha2FormatIP, + } +} + +func toTemplateNetworkStatusV1A1(bsArgs *BootstrapArgs) []v1alpha1.NetworkDeviceStatus { + networkDevicesStatus := make([]v1alpha1.NetworkDeviceStatus, 0, len(bsArgs.NetworkResults.Results)) + + for _, result := range bsArgs.NetworkResults.Results { + // When using Sysprep, the MAC address must be in the format of "-". + // CloudInit normalizes it again to ":" when adding it to the netplan. + macAddr := strings.ReplaceAll(result.MacAddress, ":", "-") + + status := v1alpha1.NetworkDeviceStatus{ + MacAddress: macAddr, + } + + for _, ipConfig := range result.IPConfigs { + // We mostly only did IPv4 before so keep that going. + if ipConfig.IsIPv4 { + if status.Gateway4 == "" { + status.Gateway4 = ipConfig.Gateway + } + + status.IPAddresses = append(status.IPAddresses, ipConfig.IPCIDR) + } + } + + networkDevicesStatus = append(networkDevicesStatus, status) + } + + return networkDevicesStatus +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go new file mode 100644 index 000000000..18c6a6b22 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go @@ -0,0 +1,221 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("TemplateVMMetadata", func() { + + const ( + ip1 = "192.168.1.37" + ip1Cidr = ip1 + "/24" + ip2 = "192.168.10.48" + ip2Cidr = ip2 + "/24" + gateway1 = "192.168.1.1" + gateway2 = "192.168.10.1" + nameserver1 = "8.8.8.8" + nameserver2 = "1.1.1.1" + macAddr1 = "8a-cb-a0-1d-8d-c4" + macAddr2 = "00-cb-30-42-05-89" + ) + + var ( + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + bsArgs *vmlifecycle.BootstrapArgs + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-vm", + Namespace: "dummy-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithName("bootstrap-template-tests"), + VM: vm, + } + + bsArgs = &vmlifecycle.BootstrapArgs{} + bsArgs.Data = make(map[string]string) + bsArgs.DNSServers = []string{nameserver1, nameserver2} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: gateway1, + IPCIDR: ip1Cidr, + IsIPv4: true, + }, + }, + }, + { + MacAddress: macAddr2, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: gateway2, + IPCIDR: ip2Cidr, + IsIPv4: true, + }, + }, + }, + } + }) + + Context("Template Functions", func() { + DescribeTable("v1alpha1 template functions", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("first_cidrIp", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}", ip1Cidr), + Entry("second_cidrIp", "{{ (index (index .V1alpha1.Net.Devices 1).IPAddresses 0) }}", ip2Cidr), + Entry("first_gateway", "{{ (index .V1alpha1.Net.Devices 0).Gateway4 }}", gateway1), + Entry("second_gateway", "{{ (index .V1alpha1.Net.Devices 1).Gateway4 }}", gateway2), + Entry("nameserver", "{{ (index .V1alpha1.Net.Nameservers 0) }}", nameserver1), + Entry("first_macAddr", "{{ (index .V1alpha1.Net.Devices 0).MacAddress }}", macAddr1), + Entry("second_macAddr", "{{ (index .V1alpha1.Net.Devices 1).MacAddress }}", macAddr2), + Entry("name", "{{ .V1alpha1.VM.Name }}", "dummy-vm"), + ) + + DescribeTable("v1alpha2 template functions", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("first_cidrIp", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}", ip1Cidr), + Entry("second_cidrIp", "{{ (index (index .V1alpha2.Net.Devices 1).IPAddresses 0) }}", ip2Cidr), + Entry("first_gateway", "{{ (index .V1alpha2.Net.Devices 0).Gateway4 }}", gateway1), + Entry("second_gateway", "{{ (index .V1alpha2.Net.Devices 1).Gateway4 }}", gateway2), + Entry("nameserver", "{{ (index .V1alpha2.Net.Nameservers 0) }}", nameserver1), + Entry("first_macAddr", "{{ (index .V1alpha2.Net.Devices 0).MacAddress }}", macAddr1), + Entry("second_macAddr", "{{ (index .V1alpha2.Net.Devices 1).MacAddress }}", macAddr2), + Entry("name", "{{ .V1alpha2.VM.Name }}", "dummy-vm"), + ) + }) + + Context("Function names", func() { + DescribeTable("v1alpha1 constant names", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("cidr_ip1", "{{ "+constants.V1alpha1FirstIP+" }}", ip1Cidr), + Entry("cidr_ip2", "{{ "+constants.V1alpha1FirstIPFromNIC+" 1 }}", ip2Cidr), + Entry("cidr_ip3", "{{ ("+constants.V1alpha1IP+" \"192.168.1.37\") }}", ip1Cidr), + Entry("cidr_ip4", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"/24\") }}", ip1Cidr), + Entry("cidr_ip5", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip6", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip7", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"/24\") }}", ip1Cidr), + Entry("ip1", "{{ "+constants.V1alpha1FormatIP+" "+constants.V1alpha1FirstIP+" \"\" }}", ip1), + Entry("ip2", "{{ "+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"\" }}", ip1), + Entry("ips_1", "{{ "+constants.V1alpha1IPsFromNIC+" 0 }}", fmt.Sprint([]string{ip1Cidr})), + Entry("subnetmask", "{{ "+constants.V1alpha1SubnetMask+" \"192.168.1.37/26\" }}", "255.255.255.192"), + Entry("firstNicMacAddr", "{{ "+constants.V1alpha1FirstNicMacAddr+" }}", macAddr1), + Entry("formatted_nameserver1", "{{ "+constants.V1alpha1FormatNameservers+" 1 \"-\"}}", nameserver1), + Entry("formatted_nameserver2", "{{ "+constants.V1alpha1FormatNameservers+" -1 \"-\"}}", nameserver1+"-"+nameserver2), + ) + + DescribeTable("v1alpha2 constant names", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("cidr_ip1", "{{ "+constants.V1alpha2FirstIP+" }}", ip1Cidr), + Entry("cidr_ip2", "{{ "+constants.V1alpha2FirstIPFromNIC+" 1 }}", ip2Cidr), + Entry("cidr_ip3", "{{ ("+constants.V1alpha2IP+" \"192.168.1.37\") }}", ip1Cidr), + Entry("cidr_ip4", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"/24\") }}", ip1Cidr), + Entry("cidr_ip5", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip6", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip7", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"/24\") }}", ip1Cidr), + Entry("ip1", "{{ "+constants.V1alpha2FormatIP+" "+constants.V1alpha1FirstIP+" \"\" }}", ip1), + Entry("ip2", "{{ "+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"\" }}", ip1), + Entry("ips_1", "{{ "+constants.V1alpha2IPsFromNIC+" 0 }}", fmt.Sprint([]string{ip1Cidr})), + Entry("subnetmask", "{{ "+constants.V1alpha2SubnetMask+" \"192.168.1.37/26\" }}", "255.255.255.192"), + Entry("firstNicMacAddr", "{{ "+constants.V1alpha2FirstNicMacAddr+" }}", macAddr1), + Entry("formatted_nameserver1", "{{ "+constants.V1alpha2FormatNameservers+" 1 \"-\"}}", nameserver1), + Entry("formatted_nameserver2", "{{ "+constants.V1alpha2FormatNameservers+" -1 \"-\"}}", nameserver1+"-"+nameserver2), + ) + }) + + Context("Invalid template names", func() { + DescribeTable("returns the original text", + func(str string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(str)) + }, + Entry("ip1", "{{ "+constants.V1alpha1IP+" \"192.1.0\" }}"), + Entry("ip2", "{{ "+constants.V1alpha1FirstIPFromNIC+" 5 }}"), + Entry("ips_1", "{{ "+constants.V1alpha1IPsFromNIC+" 5 }}"), + Entry("cidr_ip1", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"127.255.255.255\") }}"), + Entry("cidr_ip2", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1\" \"255.0.0.0\") }}"), + Entry("gateway", "{{ (index .V1alpha1.Net.NetworkInterfaces ).Gateway }}"), + Entry("nameserver", "{{ (index .V1alpha1.Net.NameServers 0) }}"), + ) + + DescribeTable("returns the original text, v1a2 style", + func(str string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(str)) + }, + Entry("ip1", "{{ "+constants.V1alpha2IP+" \"192.1.0\" }}"), + Entry("ip2", "{{ "+constants.V1alpha2FirstIPFromNIC+" 5 }}"), + Entry("ips_1", "{{ "+constants.V1alpha2IPsFromNIC+" 5 }}"), + Entry("cidr_ip1", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"127.255.255.255\") }}"), + Entry("cidr_ip2", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1\" \"255.0.0.0\") }}"), + Entry("gateway", "{{ (index .V1alpha2.Net.NetworkInterfaces ).Gateway }}"), + Entry("nameserver", "{{ (index .V1alpha2.Net.NameServers 0) }}"), + ) + }) + + Context("String has escape characters", func() { + DescribeTable("return one level of escaped removed", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("skip_data1", "\\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data2", "\\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data3", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data4", "skip \\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "skip {{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + ) + + DescribeTable("return one level of escaped removed, v1a2 style", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("skip_data1", "\\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data2", "\\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data3", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data4", "skip \\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "skip {{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + ) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go new file mode 100644 index 000000000..11d3762cc --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go @@ -0,0 +1,51 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("Customization utils", func() { + Context("IsPending", func() { + var extraConfig []vimTypes.BaseOptionValue + var pending bool + + BeforeEach(func() { + extraConfig = nil + }) + + JustBeforeEach(func() { + pending = vmlifecycle.IsCustomizationPendingExtraConfig(extraConfig) + }) + + Context("Empty ExtraConfig", func() { + It("not pending", func() { + Expect(pending).To(BeFalse()) + }) + }) + + Context("ExtraConfig with pending key", func() { + BeforeEach(func() { + extraConfig = append(extraConfig, &vimTypes.OptionValue{ + Key: constants.GOSCPendingExtraConfigKey, + Value: "/foo/bar", + }) + }) + + It("is pending", func() { + Expect(pending).To(BeTrue()) + }) + }) + }) +}) + +// TODO: We should at least a few basic DoBootstrap() tests so we test the overall +// Reconfigure/Customize flow but the old code didn't. diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go new file mode 100644 index 000000000..241f28928 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go @@ -0,0 +1,108 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +func BootstrapVAppConfig( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + + return configSpec, nil, nil +} + +func GetOVFVAppConfigForConfigSpec( + config *vimTypes.VirtualMachineConfigInfo, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + vAppData map[string]string, + vAppExData map[string]map[string]string, + templateRenderFn TemplateRenderFunc) vimTypes.BaseVmConfigSpec { + + if config.VAppConfig == nil { + // BMV: Should we really silently return here and below? + return nil + } + + vAppConfigInfo := config.VAppConfig.GetVmConfigInfo() + if vAppConfigInfo == nil { + return nil + } + + if len(vAppConfigSpec.Properties) > 0 { + vAppData = map[string]string{} + + for _, p := range vAppConfigSpec.Properties { + if p.Value.Value != nil { + vAppData[p.Key] = *p.Value.Value + } else if p.Value.From != nil { + from := p.Value.From + vAppData[p.Key] = vAppExData[from.Name][from.Key] + } + } + } + + if templateRenderFn != nil { + // If we have a templating func, apply it to whatever data we have, regardless of the source. + for k, v := range vAppData { + vAppData[k] = templateRenderFn(k, v) + } + } + + return GetMergedvAppConfigSpec(vAppData, vAppConfigInfo.Property) +} + +// GetMergedvAppConfigSpec prepares a vApp VmConfigSpec which will set the provided key/value fields. +// Only fields marked userConfigurable and pre-existing on the VM (ie. originated from the OVF Image) +// will be set, and all others will be ignored. +func GetMergedvAppConfigSpec(inProps map[string]string, vmProps []vimTypes.VAppPropertyInfo) *vimTypes.VmConfigSpec { + outProps := make([]vimTypes.VAppPropertySpec, 0) + + for _, vmProp := range vmProps { + if vmProp.UserConfigurable == nil || !*vmProp.UserConfigurable { + continue + } + + inPropValue, found := inProps[vmProp.Id] + if !found || vmProp.Value == inPropValue { + continue + } + + vmPropCopy := vmProp + vmPropCopy.Value = inPropValue + outProp := vimTypes.VAppPropertySpec{ + ArrayUpdateSpec: vimTypes.ArrayUpdateSpec{ + Operation: vimTypes.ArrayUpdateOperationEdit, + }, + Info: &vmPropCopy, + } + outProps = append(outProps, outProp) + } + + if len(outProps) == 0 { + return nil + } + + return &vimTypes.VmConfigSpec{ + Property: outProps, + // Ensure the transport is guestInfo in case the VM does not have + // a CD-ROM device required to use the ISO transport. + OvfEnvironmentTransport: []string{OvfEnvironmentTransportGuestInfo}, + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go new file mode 100644 index 000000000..6533b3f6b --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go @@ -0,0 +1,267 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("VAppConfig Bootstrap", func() { + const key, value = "fooKey", "fooValue" + + var ( + configInfo *types.VirtualMachineConfigInfo + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + bsArgs vmlifecycle.BootstrapArgs + baseVMConfigSpec types.BaseVmConfigSpec + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + bsArgs.VAppData = make(map[string]string) + bsArgs.VAppExData = make(map[string]map[string]string) + }) + + AfterEach(func() { + vAppConfigSpec = nil + baseVMConfigSpec = nil + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("GetOVFVAppConfigForConfigSpec", func() { + + JustBeforeEach(func() { + baseVMConfigSpec = vmlifecycle.GetOVFVAppConfigForConfigSpec( + configInfo, + vAppConfigSpec, + bsArgs.VAppData, + bsArgs.VAppExData, + bsArgs.TemplateRenderFn) + }) + + Context("Empty input", func() { + It("No changes", func() { + Expect(baseVMConfigSpec).To(BeNil()) + }) + }) + + Context("vAppData Map", func() { + BeforeEach(func() { + bsArgs.VAppData[key] = value + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + + Context("vAppDataConfig Inlined Properties", func() { + BeforeEach(func() { + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + + Context("vAppDataConfig From Properties", func() { + const secretName = "my-other-secret" + + BeforeEach(func() { + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{ + From: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: key, + Optional: nil, // TODO: Rethink if we really need this complexity + }, + }, + }, + }, + } + + bsArgs.VAppExData[secretName] = map[string]string{key: value} + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + }) +}) + +var _ = Describe("GetMergedvAppConfigSpec", func() { + + DescribeTable("returns expected props", + func(inProps map[string]string, vmProps []types.VAppPropertyInfo, expected *types.VmConfigSpec) { + vAppConfigSpec := vmlifecycle.GetMergedvAppConfigSpec(inProps, vmProps) + if expected == nil { + Expect(vAppConfigSpec).To(BeNil()) + } else { + Expect(vAppConfigSpec.Property).To(HaveLen(len(expected.Property))) + for i := range vAppConfigSpec.Property { + Expect(vAppConfigSpec.Property[i].Info.Key).To(Equal(expected.Property[i].Info.Key)) + Expect(vAppConfigSpec.Property[i].Info.Id).To(Equal(expected.Property[i].Info.Id)) + Expect(vAppConfigSpec.Property[i].Info.Value).To(Equal(expected.Property[i].Info.Value)) + Expect(vAppConfigSpec.Property[i].ArrayUpdateSpec.Operation).To(Equal(types.ArrayUpdateOperationEdit)) + } + Expect(vAppConfigSpec.OvfEnvironmentTransport).To(HaveLen(1)) + Expect(vAppConfigSpec.OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + } + }, + Entry("return nil for absent vm and input props", + map[string]string{}, + []types.VAppPropertyInfo{}, + nil, + ), + Entry("return nil for non UserConfigurable vm props", + map[string]string{ + "one-id": "one-override-value", + "two-id": "two-override-value", + }, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value"}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(false)}, + }, + nil, + ), + Entry("return nil for UserConfigurable vm props but no input props", + map[string]string{}, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value"}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(true)}, + }, + nil, + ), + Entry("return valid vAppConfigSpec for setting mixed UserConfigurable props", + map[string]string{ + "one-id": "one-override-value", + "two-id": "two-override-value", + "three-id": "three-override-value", + }, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value", UserConfigurable: nil}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(true)}, + {Key: 3, Id: "three-id", Value: "three-value", UserConfigurable: pointer.Bool(false)}, + }, + &types.VmConfigSpec{ + Property: []types.VAppPropertySpec{ + {Info: &types.VAppPropertyInfo{Key: 2, Id: "two-id", Value: "two-override-value"}}, + }, + }, + ), + ) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go new file mode 100644 index 000000000..a6da98006 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go @@ -0,0 +1,41 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +// CreateArgs contains the arguments needed to create a VM. +type CreateArgs struct { + UseContentLibrary bool + ProviderItemID string + + ConfigSpec *types.VirtualMachineConfigSpec + StorageProvisioning string + FolderMoID string + ResourcePoolMoID string + HostMoID string + StorageProfileID string + DatastoreMoID string // gce2e only: used only if StorageProfileID is unset +} + +func CreateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + clClient contentlibrary.Provider, + restClient *rest.Client, + finder *find.Finder, + createArgs *CreateArgs) (*types.ManagedObjectReference, error) { + + if createArgs.UseContentLibrary { + return deployFromContentLibrary(vmCtx, clClient, restClient, createArgs) + } + + return cloneVMFromInventory(vmCtx, finder, createArgs) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go new file mode 100644 index 000000000..299d0bba7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go @@ -0,0 +1,188 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + vimtypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/placement" +) + +// CloneVMFromInventory creates a new VM by cloning the source VM. This is not reachable/used +// in production because we only really support deploying an OVF via content library. +// Maybe someday we'll use clone to speed up VM deployment so keep this code around and unit tested. +func cloneVMFromInventory( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + srcVMName := createArgs.ProviderItemID // AKA: vmCtx.VM.Spec.ImageName + + srcVM, err := finder.VirtualMachine(vmCtx, srcVMName) + if err != nil { + return nil, errors.Wrapf(err, "failed to find clone source VM: %s", srcVMName) + } + + cloneSpec, err := createCloneSpec(vmCtx, createArgs, srcVM) + if err != nil { + return nil, errors.Wrap(err, "failed to create CloneSpec") + } + + // We always set cloneSpec.Location.Folder so use that to get the parent folder object. + folder := object.NewFolder(srcVM.Client(), *cloneSpec.Location.Folder) + + cloneTask, err := srcVM.Clone(vmCtx, folder, cloneSpec.Config.Name, *cloneSpec) + if err != nil { + return nil, err + } + + result, err := cloneTask.WaitForResult(vmCtx, nil) + if err != nil { + return nil, errors.Wrapf(err, "clone VM task failed") + } + + ref := result.Result.(vimtypes.ManagedObjectReference) + return &ref, nil +} + +func createCloneSpec( + vmCtx context.VirtualMachineContextA2, + createArgs *CreateArgs, + srcVM *object.VirtualMachine) (*vimtypes.VirtualMachineCloneSpec, error) { + + cloneSpec := &vimtypes.VirtualMachineCloneSpec{ + Config: createArgs.ConfigSpec, + Memory: pointer.Bool(false), // No full memory clones. + } + + virtualDevices, err := srcVM.Device(vmCtx) + if err != nil { + return nil, fmt.Errorf("failed to get clone source VM devices: %w", err) + } + + virtualDisks := virtualDevices.SelectByType((*vimtypes.VirtualDisk)(nil)) + + for _, deviceChange := range resizeBootDiskDeviceChange(vmCtx, virtualDisks) { + if deviceChange.GetVirtualDeviceConfigSpec().Operation == vimtypes.VirtualDeviceConfigSpecOperationEdit { + cloneSpec.Location.DeviceChange = append(cloneSpec.Location.DeviceChange, deviceChange) + } else { + cloneSpec.Config.DeviceChange = append(cloneSpec.Config.DeviceChange, deviceChange) + } + } + + if createArgs.StorageProfileID != "" { + cloneSpec.Location.Profile = []vimtypes.BaseVirtualMachineProfileSpec{ + &vimtypes.VirtualMachineDefinedProfileSpec{ProfileId: createArgs.StorageProfileID}, + } + } else { + // BMV: Used to compute placement? Otherwise, always overwritten later. + cloneSpec.Location.Datastore = &vimtypes.ManagedObjectReference{ + Type: "Datastore", + Value: createArgs.DatastoreMoID, + } + } + + cloneSpec.Location.Pool = &vimtypes.ManagedObjectReference{ + Type: "ResourcePool", + Value: createArgs.ResourcePoolMoID, + } + cloneSpec.Location.Folder = &vimtypes.ManagedObjectReference{ + Type: "Folder", + Value: createArgs.FolderMoID, + } + + rpOwner, err := object.NewResourcePool(srcVM.Client(), *cloneSpec.Location.Pool).Owner(vmCtx) + if err != nil { + return nil, err + } + + cluster, ok := rpOwner.(*object.ClusterComputeResource) + if !ok { + return nil, fmt.Errorf("owner of the ResourcePool is not a cluster but %T", rpOwner) + } + + relocateSpec, err := placement.CloneVMRelocateSpec(vmCtx, cluster, srcVM.Reference(), cloneSpec) + if err != nil { + return nil, err + } + + cloneSpec.Location.Host = relocateSpec.Host + cloneSpec.Location.Datastore = relocateSpec.Datastore + cloneSpec.Location.Disk = cloneVMDiskLocators(virtualDisks, createArgs, cloneSpec.Location) + + return cloneSpec, nil +} + +func cloneVMDiskLocators( + disks object.VirtualDeviceList, + createArgs *CreateArgs, + location vimtypes.VirtualMachineRelocateSpec) []vimtypes.VirtualMachineRelocateSpecDiskLocator { + + diskLocators := make([]vimtypes.VirtualMachineRelocateSpecDiskLocator, 0, len(disks)) + + for _, disk := range disks { + locator := vimtypes.VirtualMachineRelocateSpecDiskLocator{ + DiskId: disk.GetVirtualDevice().Key, + Datastore: *location.Datastore, + Profile: location.Profile, + // TODO: Check if policy is encrypted and use correct DiskMoveType + DiskMoveType: string(vimtypes.VirtualMachineRelocateDiskMoveOptionsMoveChildMostDiskBacking), + } + + if backing, ok := disk.(*vimtypes.VirtualDisk).Backing.(*vimtypes.VirtualDiskFlatVer2BackingInfo); ok { + switch createArgs.StorageProvisioning { + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeThin): + backing.ThinProvisioned = pointer.Bool(true) + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeThick): + backing.ThinProvisioned = pointer.Bool(false) + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeEagerZeroedThick): + backing.EagerlyScrub = pointer.Bool(true) + } + locator.DiskBackingInfo = backing + } + + diskLocators = append(diskLocators, locator) + } + + return diskLocators +} + +func resizeBootDiskDeviceChange( + vmCtx context.VirtualMachineContextA2, + virtualDisks object.VirtualDeviceList) []vimtypes.BaseVirtualDeviceConfigSpec { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil + } + + // Assume the first virtual disk - if any - is the boot disk. + var deviceChanges []vimtypes.BaseVirtualDeviceConfigSpec + for _, vmDevice := range virtualDisks { + vmDisk, ok := vmDevice.(*vimtypes.VirtualDisk) + if !ok { + continue + } + + // Maybe don't allow shrink? + if vmDisk.CapacityInBytes != capacity.Value() { + vmDisk.CapacityInBytes = capacity.Value() + deviceChanges = append(deviceChanges, &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationEdit, + Device: vmDisk, + }) + } + + break + } + + return deviceChanges +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go new file mode 100644 index 000000000..e0edefb55 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go @@ -0,0 +1,105 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "encoding/base64" + "fmt" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vapi/vcenter" + vimtypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +func deployOVF( + vmCtx context.VirtualMachineContextA2, + restClient *rest.Client, + item *library.Item, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + deploymentSpec := vcenter.DeploymentSpec{ + Name: vmCtx.VM.Name, + StorageProfileID: createArgs.StorageProfileID, + StorageProvisioning: createArgs.StorageProvisioning, + AcceptAllEULA: true, + } + + if deploymentSpec.StorageProfileID == "" { + // Without a storage profile, fall back to the datastore. + deploymentSpec.DefaultDatastoreID = createArgs.DatastoreMoID + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && createArgs.ConfigSpec != nil { + configSpecXML, err := util.MarshalConfigSpecToXML(createArgs.ConfigSpec) + if err != nil { + return nil, fmt.Errorf("failed to marshal ConfigSpec to XML: %w", err) + } + + deploymentSpec.VmConfigSpec = &vcenter.VmConfigSpec{ + Provider: constants.ConfigSpecProviderXML, + XML: base64.StdEncoding.EncodeToString(configSpecXML), + } + } + + deploy := vcenter.Deploy{ + DeploymentSpec: deploymentSpec, + Target: vcenter.Target{ + ResourcePoolID: createArgs.ResourcePoolMoID, + FolderID: createArgs.FolderMoID, + HostID: createArgs.HostMoID, + }, + } + + vmCtx.Logger.Info("Deploying OVF Library Item", "itemID", item.ID, "itemName", item.Name, "deploy", deploy) + + return vcenter.NewManager(restClient).DeployLibraryItem(vmCtx, item.ID, deploy) +} + +func deployVMTX( + vmCtx context.VirtualMachineContextA2, + restClient *rest.Client, + item *library.Item, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + // Not yet supported. This item type needs to be deployed via DeployTemplateLibraryItem(), + // which doesn't take a ConfigSpec so it is a heavy lift. + // TODO: We should catch this earlier to avoid a bunch of wasted work. + + _ = vmCtx + _ = restClient + _ = createArgs + + return nil, fmt.Errorf("creating VM from VMTX content library type is not supported: %s", item.Name) +} + +func deployFromContentLibrary( + vmCtx context.VirtualMachineContextA2, + clClient contentlibrary.Provider, + restClient *rest.Client, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + // This call is needed to get the item type. We could avoid going to CL here, and + // instead get the item type via the {Cluster}ContentLibrary CR for the image. + item, err := clClient.GetLibraryItemID(vmCtx, createArgs.ProviderItemID) + if err != nil { + return nil, err + } + + switch item.Type { + case library.ItemTypeOVF: + return deployOVF(vmCtx, restClient, item, createArgs) + case library.ItemTypeVMTX: + return deployVMTX(vmCtx, restClient, item, createArgs) + default: + return nil, errors.Errorf("item %s not a supported type: %s", item.Name, item.Type) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go new file mode 100644 index 000000000..62bf35ba4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go @@ -0,0 +1,345 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + "net" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var ( + // The minimum properties needed to be retrieved in order to populate the Status. Callers may + // provide a MO with more. This often saves us a second round trip in the common steady state. + vmStatusPropertiesSelector = []string{"config.changeTrackingEnabled", "guest", "summary"} +) + +func UpdateStatus( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vcVM *object.VirtualMachine, + vmMO *mo.VirtualMachine) error { + + vm := vmCtx.VM + + // This is implicitly true: ensure the condition is set since it is how we determine the old v1a1 Phase. + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionCreated) + // TODO: Might set other "prereq" conditions too for version conversion but we'd have to fib a little. + + if vm.Status.Image == nil { + // If unset, we don't know if this was a cluster or namespace scoped image at the time. + vm.Status.Image = &common.LocalObjectRef{Name: vm.Spec.ImageName} + } + if vm.Status.Class == nil { + // We can most likely just assume the other fields of the LocalObjectRef but only fill + // in the Name for now. Our handling of this field will be more complicated once we really + // support class changes and reconfiguring the VM the fly in response. + vm.Status.Class = &common.LocalObjectRef{Name: vm.Spec.ClassName} + } + + if vmMO == nil { + // In the common case, our caller will have already gotten the MO properties in order to determine + // if it had any reconciliation to do, and there was nothing to do since the VM is in the steady + // state so that MO is still entirely valid here. + // NOTE: The properties must have been retrieved with at least vmStatusPropertiesSelector. + vmMO = &mo.VirtualMachine{} + if err := vcVM.Properties(vmCtx, vcVM.Reference(), vmStatusPropertiesSelector, vmMO); err != nil { + // Leave the current Status unchanged for now. + return fmt.Errorf("failed to get VM properties for status update: %w", err) + } + } + + var errs []error + var err error + summary := vmMO.Summary + + vm.Status.PowerState = convertPowerState(summary.Runtime.PowerState) + vm.Status.UniqueID = vcVM.Reference().Value + vm.Status.BiosUUID = summary.Config.Uuid + vm.Status.InstanceUUID = summary.Config.InstanceUuid + vm.Status.Network = getGuestNetworkStatus(vmMO.Guest) + + vm.Status.Host, err = getRuntimeHostHostname(vmCtx, vcVM, summary.Runtime.Host) + if err != nil { + errs = append(errs, err) + } + + MarkVMToolsRunningStatusCondition(vmCtx.VM, vmMO.Guest) + MarkCustomizationInfoCondition(vmCtx.VM, vmMO.Guest) + + if config := vmMO.Config; config != nil { + vm.Status.ChangeBlockTracking = config.ChangeTrackingEnabled + } else { + vm.Status.ChangeBlockTracking = nil + } + + if lib.IsWcpFaultDomainsFSSEnabled() { + zoneName := vm.Labels[topology.KubernetesTopologyZoneLabelKey] + if zoneName == "" { + cluster, err := virtualmachine.GetVMClusterComputeResource(vmCtx, vcVM) + if err != nil { + errs = append(errs, err) + } else { + zoneName, err = topology.LookupZoneForClusterMoID(vmCtx, k8sClient, cluster.Reference().Value) + if err != nil { + errs = append(errs, err) + } else { + if vm.Labels == nil { + vm.Labels = map[string]string{} + } + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + } + } + } + + if zoneName != "" { + vm.Status.Zone = zoneName + } + } + + return k8serrors.NewAggregate(errs) +} + +func getRuntimeHostHostname( + ctx goctx.Context, + vcVM *object.VirtualMachine, + host *types.ManagedObjectReference) (string, error) { + + if host != nil { + return object.NewHostSystem(vcVM.Client(), *host).ObjectName(ctx) + } + return "", nil +} + +func getGuestNetworkStatus(guestInfo *types.GuestInfo) *vmopv1.VirtualMachineNetworkStatus { + if guestInfo == nil { + return nil + } + + status := &vmopv1.VirtualMachineNetworkStatus{} + + if ipAddr := guestInfo.IpAddress; ipAddr != "" { + // TODO: Filter out local addresses. + if net.ParseIP(ipAddr).To4() != nil { + status.PrimaryIP4 = ipAddr + } else { + status.PrimaryIP6 = ipAddr + } + } + + if len(guestInfo.Net) > 0 { + status.Interfaces = make([]vmopv1.VirtualMachineNetworkInterfaceStatus, 0, len(guestInfo.Net)) + for i := range guestInfo.Net { + status.Interfaces = append(status.Interfaces, guestNicInfoToInterfaceStatus(i, &guestInfo.Net[i])) + } + } + + if len(guestInfo.IpStack) > 0 { + status.VirtualMachineNetworkIPStackStatus = guestIPStackInfoToIPStackStatus(&guestInfo.IpStack[0]) + } + + return status +} + +func guestNicInfoToInterfaceStatus(idx int, guestNicInfo *types.GuestNicInfo) vmopv1.VirtualMachineNetworkInterfaceStatus { + status := vmopv1.VirtualMachineNetworkInterfaceStatus{} + + // TODO: What name exactly? The CRD name may be the most useful here but hard to line that up. + // BMV: DeviceConfigId will be -1 for our pseudo-y interfaces. Most likely want to just skip those devices. + status.Name = fmt.Sprintf("nic-%d-%d", idx, guestNicInfo.DeviceConfigId) + status.IP.MACAddr = guestNicInfo.MacAddress + + if guestIPConfig := guestNicInfo.IpConfig; guestIPConfig != nil { + ip := &status.IP + + ip.AutoConfigurationEnabled = guestIPConfig.AutoConfigurationEnabled + ip.Addresses = convertNetIPConfigInfoIPAddresses(guestIPConfig.IpAddress) + + if guestIPConfig.Dhcp != nil { + ip.DHCP = convertNetDhcpConfigInfo(guestIPConfig.Dhcp) + } + } + + if dnsConfig := guestNicInfo.DnsConfig; dnsConfig != nil { + status.DNS = convertNetDNSConfigInfo(dnsConfig) + } + + return status +} + +func guestIPStackInfoToIPStackStatus(guestIPStack *types.GuestStackInfo) vmopv1.VirtualMachineNetworkIPStackStatus { + status := vmopv1.VirtualMachineNetworkIPStackStatus{} + + if dhcpConfig := guestIPStack.DhcpConfig; dhcpConfig != nil { + status.DHCP = convertNetDhcpConfigInfo(dhcpConfig) + } + + if dnsConfig := guestIPStack.DnsConfig; dnsConfig != nil { + status.DNS = convertNetDNSConfigInfo(dnsConfig) + } + + if ipRouteConfig := guestIPStack.IpRouteConfig; ipRouteConfig != nil { + status.IPRoutes = convertNetIPRouteConfigInfo(ipRouteConfig) + } + + status.KernelConfig = convertKeyValueSlice(guestIPStack.IpStackConfig) + + return status +} + +func convertPowerState(powerState types.VirtualMachinePowerState) vmopv1.VirtualMachinePowerState { + switch powerState { + case types.VirtualMachinePowerStatePoweredOff: + return vmopv1.VirtualMachinePowerStateOff + case types.VirtualMachinePowerStatePoweredOn: + return vmopv1.VirtualMachinePowerStateOn + case types.VirtualMachinePowerStateSuspended: + return vmopv1.VirtualMachinePowerStateSuspended + } + return "" +} + +func convertNetIPConfigInfoIPAddresses(ipAddresses []types.NetIpConfigInfoIpAddress) []vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus { + if len(ipAddresses) == 0 { + return nil + } + + out := make([]vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus, 0, len(ipAddresses)) + for _, guestIPAddr := range ipAddresses { + ipAddrStatus := vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus{ + Address: guestIPAddr.IpAddress, + Origin: guestIPAddr.Origin, + State: guestIPAddr.State, + } + if guestIPAddr.Lifetime != nil { + ipAddrStatus.Lifetime = metav1.NewTime(*guestIPAddr.Lifetime) + } + + out = append(out, ipAddrStatus) + } + return out +} + +func convertNetDNSConfigInfo(dnsConfig *types.NetDnsConfigInfo) vmopv1.VirtualMachineNetworkDNSStatus { + return vmopv1.VirtualMachineNetworkDNSStatus{ + DHCP: dnsConfig.Dhcp, + DomainName: dnsConfig.DomainName, + HostName: dnsConfig.HostName, + Nameservers: dnsConfig.IpAddress, + SearchDomains: dnsConfig.SearchDomain, + } +} + +func convertNetDhcpConfigInfo(dhcpConfig *types.NetDhcpConfigInfo) vmopv1.VirtualMachineNetworkDHCPStatus { + status := vmopv1.VirtualMachineNetworkDHCPStatus{} + + if ipv4 := dhcpConfig.Ipv4; ipv4 != nil { + status.IP4.Enabled = ipv4.Enable + status.IP4.Config = convertKeyValueSlice(ipv4.Config) + } + + if ipv6 := dhcpConfig.Ipv6; ipv6 != nil { + status.IP6.Enabled = ipv6.Enable + status.IP6.Config = convertKeyValueSlice(ipv6.Config) + } + + return status +} + +func convertNetIPRouteConfigInfo(routeConfig *types.NetIpRouteConfigInfo) []vmopv1.VirtualMachineNetworkIPRouteStatus { + if len(routeConfig.IpRoute) == 0 { + return nil + } + + // TODO: Prob only want to show default routes. Will be very verbose on TKG nodes. + out := make([]vmopv1.VirtualMachineNetworkIPRouteStatus, 0, len(routeConfig.IpRoute)) + for _, ipRoute := range routeConfig.IpRoute { + out = append(out, vmopv1.VirtualMachineNetworkIPRouteStatus{ + Gateway: vmopv1.VirtualMachineNetworkIPRouteGatewayStatus{ + Device: ipRoute.Gateway.Device, + Address: ipRoute.Gateway.IpAddress, + }, + NetworkAddress: fmt.Sprintf("%s/%d", ipRoute.Network, ipRoute.PrefixLength), + }) + } + return out +} + +func convertKeyValueSlice(s []types.KeyValue) []common.KeyValuePair { + if len(s) == 0 { + return nil + } + + out := make([]common.KeyValuePair, 0, len(s)) + for i := range s { + out = append(out, common.KeyValuePair{Key: s[i].Key, Value: s[i].Value}) + } + return out +} + +func MarkVMToolsRunningStatusCondition( + vm *vmopv1.VirtualMachine, + guestInfo *types.GuestInfo) { + + if guestInfo == nil || guestInfo.ToolsRunningStatus == "" { + conditions.MarkUnknown(vm, vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", "") + return + } + + switch guestInfo.ToolsRunningStatus { + case string(types.VirtualMachineToolsRunningStatusGuestToolsNotRunning): + msg := "VMware Tools is not running" + conditions.MarkFalse(vm, vmopv1.VirtualMachineToolsCondition, vmopv1.VirtualMachineToolsNotRunningReason, msg) + case string(types.VirtualMachineToolsRunningStatusGuestToolsRunning), string(types.VirtualMachineToolsRunningStatusGuestToolsExecutingScripts): + conditions.MarkTrue(vm, vmopv1.VirtualMachineToolsCondition) + default: + msg := "Unexpected VMware Tools running status" + conditions.MarkUnknown(vm, vmopv1.VirtualMachineToolsCondition, "Unknown", msg) + } +} + +func MarkCustomizationInfoCondition(vm *vmopv1.VirtualMachine, guestInfo *types.GuestInfo) { + if guestInfo == nil || guestInfo.CustomizationInfo == nil { + conditions.MarkUnknown(vm, vmopv1.GuestCustomizationCondition, "NoGuestInfo", "") + return + } + + switch guestInfo.CustomizationInfo.CustomizationStatus { + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_IDLE), "": + conditions.MarkTrue(vm, vmopv1.GuestCustomizationCondition) + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_PENDING): + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationPendingReason, "") + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_RUNNING): + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationRunningReason, "") + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_SUCCEEDED): + conditions.MarkTrue(vm, vmopv1.GuestCustomizationCondition) + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_FAILED): + errorMsg := guestInfo.CustomizationInfo.ErrorMsg + if errorMsg == "" { + errorMsg = "vSphere VM Customization failed due to an unknown error." + } + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationFailedReason, errorMsg) + default: + errorMsg := guestInfo.CustomizationInfo.ErrorMsg + if errorMsg == "" { + errorMsg = "Unexpected VM Customization status" + } + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, "Unknown", errorMsg) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go new file mode 100644 index 000000000..3d130cdc9 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("VirtualMachineTools Status to VM Status Condition", func() { + Context("markVMToolsRunningStatusCondition", func() { + var ( + vm *vmopv1.VirtualMachine + guestInfo *types.GuestInfo + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{} + guestInfo = &types.GuestInfo{ + ToolsRunningStatus: "", + } + }) + + JustBeforeEach(func() { + vmlifecycle.MarkVMToolsRunningStatusCondition(vm, guestInfo) + }) + + Context("guestInfo is nil", func() { + BeforeEach(func() { + guestInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("ToolsRunningStatus is empty", func() { + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is not running", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsNotRunning) + }) + It("sets condition to false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineToolsCondition, vmopv1.VirtualMachineToolsNotRunningReason, "VMware Tools is not running"), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is running", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsRunning) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.VirtualMachineToolsCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is starting", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsExecutingScripts) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.VirtualMachineToolsCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("Unexpected vmtools running status", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = "blah" + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "Unknown", "Unexpected VMware Tools running status"), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + }) +}) + +var _ = Describe("VSphere Customization Status to VM Status Condition", func() { + Context("markCustomizationInfoCondition", func() { + var ( + vm *vmopv1.VirtualMachine + guestInfo *types.GuestInfo + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{} + guestInfo = &types.GuestInfo{ + CustomizationInfo: &types.GuestInfoCustomizationInfo{}, + } + }) + + JustBeforeEach(func() { + vmlifecycle.MarkCustomizationInfoCondition(vm, guestInfo) + }) + + Context("guestInfo unset", func() { + BeforeEach(func() { + guestInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.GuestCustomizationCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo unset", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.GuestCustomizationCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo idle", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_IDLE) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.GuestCustomizationCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo pending", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_PENDING) + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationPendingReason, ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo running", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_RUNNING) + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationRunningReason, ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo succeeded", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_SUCCEEDED) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.GuestCustomizationCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo failed", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_FAILED) + guestInfo.CustomizationInfo.ErrorMsg = "some error message" + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationFailedReason, guestInfo.CustomizationInfo.ErrorMsg), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo invalid", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = "asdf" + guestInfo.CustomizationInfo.ErrorMsg = "some error message" + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, "Unknown", guestInfo.CustomizationInfo.ErrorMsg), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go new file mode 100644 index 000000000..461db366f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestVMLifecycle(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider VM Lifecycle Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider.go b/pkg/vmprovider/providers/vsphere2/vmprovider.go new file mode 100644 index 000000000..161b24262 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider.go @@ -0,0 +1,428 @@ +// Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + goctx "context" + "encoding/json" + "fmt" + "math/rand" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/task" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/record" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + vcconfig "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +const ( + VsphereVMProviderName = "vsphere" + + // taskHistoryCollectorPageSize represents the max count to read from task manager in one iteration. + taskHistoryCollectorPageSize = 10 + + ovfCacheMaxItem = 100 + ovfCacheItemExpiration = 30 * time.Minute + ovfCacheExpirationCheckInterval = 5 * time.Minute +) + +var log = logf.Log.WithName(VsphereVMProviderName) + +type VersionedOVFEnvelope struct { + OvfEnvelope *ovf.Envelope + ContentVersion string +} + +type vSphereVMProvider struct { + k8sClient ctrlruntime.Client + eventRecorder record.Recorder + globalExtraConfig map[string]string + minCPUFreq uint64 + ovfCache *util.Cache[VersionedOVFEnvelope] + ovfCacheLockPool *util.LockPool[string, *sync.RWMutex] + + vcClientLock sync.Mutex + vcClient *vcclient.Client +} + +func NewVSphereVMProviderFromClient( + client ctrlruntime.Client, + recorder record.Recorder) vmprovider.VirtualMachineProviderInterfaceA2 { + + ovfCache, ovfLockPool := InitOvfCacheAndLockPool( + ovfCacheItemExpiration, ovfCacheExpirationCheckInterval, ovfCacheMaxItem) + + return &vSphereVMProvider{ + k8sClient: client, + eventRecorder: recorder, + globalExtraConfig: getExtraConfig(), + ovfCache: ovfCache, + ovfCacheLockPool: ovfLockPool, + } +} + +// InitOvfCacheAndLockPool initializes the ovf cache and lock pool that are used +// to cache the ovf envelope and lock the ovf envelope when it is being downloaded. +func InitOvfCacheAndLockPool(expireAfter, checkExpireInterval time.Duration, maxItems int) ( + *util.Cache[VersionedOVFEnvelope], *util.LockPool[string, *sync.RWMutex]) { + ovfCache := util.NewCache[VersionedOVFEnvelope](expireAfter, checkExpireInterval, maxItems) + ovfLockPool := &util.LockPool[string, *sync.RWMutex]{} + + // Clean up the lock pool when the ovf cache item expires. + expiredChan := ovfCache.ExpiredChan() + go func() { + for k := range expiredChan { + l := ovfLockPool.Get(k) + // This could still delete an in-use lock if it's retrieved from the pool but not locked yet. + // If it's already locked, this will wait until it's unlocked to delete it from the pool. + l.Lock() + ovfLockPool.Delete(k) + l.Unlock() + } + }() + + return ovfCache, ovfLockPool +} + +func getExtraConfig() map[string]string { + ec := map[string]string{ + constants.EnableDiskUUIDExtraConfigKey: constants.ExtraConfigTrue, + constants.GOSCIgnoreToolsCheckExtraConfigKey: constants.ExtraConfigTrue, + } + + if jsonEC := os.Getenv("JSON_EXTRA_CONFIG"); jsonEC != "" { + extraConfig := make(map[string]string) + + if err := json.Unmarshal([]byte(jsonEC), &extraConfig); err != nil { + // This is only set in testing so make errors fatal. + panic(fmt.Sprintf("invalid JSON_EXTRA_CONFIG envvar: %q %v", jsonEC, err)) + } + + for k, v := range extraConfig { + ec[k] = v + } + } + + return ec +} + +func (vs *vSphereVMProvider) getVcClient(ctx goctx.Context) (*vcclient.Client, error) { + vs.vcClientLock.Lock() + defer vs.vcClientLock.Unlock() + + if vs.vcClient != nil { + return vs.vcClient, nil + } + + config, err := vcconfig.GetProviderConfig(ctx, vs.k8sClient) + if err != nil { + return nil, err + } + + vcClient, err := vcclient.NewClient(ctx, config) + if err != nil { + return nil, err + } + + vs.vcClient = vcClient + return vcClient, nil +} + +func (vs *vSphereVMProvider) UpdateVcPNID(ctx goctx.Context, vcPNID, vcPort string) error { + updated, err := vcconfig.UpdateVcInConfigMap(ctx, vs.k8sClient, vcPNID, vcPort) + if err != nil || !updated { + return err + } + + // Our controller-runtime client does not cache ConfigMaps & Secrets, so the next time + // getVcClient() is called, it will fetch newly updated CM. + vs.clearAndLogoutVcClient(ctx) + return nil +} + +func (vs *vSphereVMProvider) ResetVcClient(ctx goctx.Context) { + vs.clearAndLogoutVcClient(ctx) +} + +func (vs *vSphereVMProvider) clearAndLogoutVcClient(ctx goctx.Context) { + vs.vcClientLock.Lock() + vcClient := vs.vcClient + vs.vcClient = nil + vs.vcClientLock.Unlock() + + if vcClient != nil { + vcClient.Logout(ctx) + } +} + +// SyncVirtualMachineImage syncs the vmi object with the OVF Envelope retrieved from the cli object. +func (vs *vSphereVMProvider) SyncVirtualMachineImage(ctx goctx.Context, cli, vmi ctrlruntime.Object) error { + var itemID, contentVersion string + switch cli := cli.(type) { + case *imgregv1a1.ContentLibraryItem: + itemID = string(cli.Spec.UUID) + contentVersion = cli.Status.ContentVersion + case *imgregv1a1.ClusterContentLibraryItem: + itemID = string(cli.Spec.UUID) + contentVersion = cli.Status.ContentVersion + default: + return errors.Errorf("unexpected content library item type %T", cli) + } + + ovfEnvelope, err := vs.getOvfEnvelope(ctx, itemID, contentVersion) + if err != nil { + return err + } + + logger := log.V(4).WithValues("vmiName", vmi.GetName(), "cliName", cli.GetName()) + if ovfEnvelope == nil { + logger.Error(nil, "skip syncing VMI as corresponding OVF envelope is nil") + return nil + } + + contentlibrary.UpdateVmiWithOvfEnvelope(vmi, *ovfEnvelope) + return nil +} + +// getOvfEnvelope gets the OVF envelope from the cache if it exists and matches version. +// If not, it downloads the OVF envelope from vCenter and stores it in the cache. +func (vs *vSphereVMProvider) getOvfEnvelope( + ctx goctx.Context, itemID, contentVersion string) (*ovf.Envelope, error) { + logger := log.V(4).WithValues("itemID", itemID, "contentVersion", contentVersion) + + // Lock the current item to prevent concurrent downloads of the same OVF. + // This is done before the get from cache below to prevent stale result. + curItemLock := vs.ovfCacheLockPool.Get(itemID) + curItemLock.Lock() + defer curItemLock.Unlock() + + isHitFunc := func(cacheItem VersionedOVFEnvelope) bool { + return cacheItem.ContentVersion == contentVersion + } + cacheItem, found := vs.ovfCache.Get(itemID, isHitFunc) + if found { + logger.Info("Cache item hit, using cached OVF") + } else { + logger.Info("Cache item miss, downloading OVF from vCenter") + client, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + ovfEnvelope, err := client.ContentLibClient().RetrieveOvfEnvelopeByLibraryItemID(ctx, itemID) + if err != nil || ovfEnvelope == nil { + return nil, err + } + + cacheItem = VersionedOVFEnvelope{ + ContentVersion: contentVersion, + OvfEnvelope: ovfEnvelope, + } + putResult := vs.ovfCache.Put(itemID, cacheItem) + logger.Info("Cache item put", "itemID", itemID, "putResult", putResult) + } + + return cacheItem.OvfEnvelope, nil +} + +// GetItemFromLibraryByName get the library item from specified content library by its name. +// Do not return error if the item doesn't exist in the content library. +func (vs *vSphereVMProvider) GetItemFromLibraryByName(ctx goctx.Context, + contentLibrary, itemName string) (*library.Item, error) { + log.V(4).Info("Get item from ContentLibrary", + "UUID", contentLibrary, "item name", itemName) + + client, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + return client.ContentLibClient().GetLibraryItem(ctx, contentLibrary, itemName, false) +} + +func (vs *vSphereVMProvider) UpdateContentLibraryItem(ctx goctx.Context, itemID, newName string, newDescription *string) error { + log.V(4).Info("Update Content Library Item", "itemID", itemID) + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + return client.ContentLibClient().UpdateLibraryItem(ctx, itemID, newName, newDescription) +} + +func (vs *vSphereVMProvider) getOpID(vm *vmopv1.VirtualMachine, operation string) string { + const charset = "0123456789abcdef" + + id := make([]byte, 8) + for i := range id { + idx := rand.Intn(len(charset)) //nolint:gosec + id[i] = charset[idx] + } + + return strings.Join([]string{"vmoperator", vm.Name, operation, string(id)}, "-") +} + +func (vs *vSphereVMProvider) getVM( + vmCtx context.VirtualMachineContextA2, + client *vcclient.Client, + notFoundReturnErr bool) (*object.VirtualMachine, error) { + + vcVM, err := vcenter.GetVirtualMachine(vmCtx, vs.k8sClient, client.VimClient(), client.Datacenter(), client.Finder()) + if err != nil { + return nil, err + } + + if vcVM == nil && notFoundReturnErr { + return nil, fmt.Errorf("VirtualMachine %q was not found on VC", vmCtx.VM.Name) + } + + return vcVM, nil +} + +func (vs *vSphereVMProvider) getOrComputeCPUMinFrequency(ctx goctx.Context) (uint64, error) { + minFreq := atomic.LoadUint64(&vs.minCPUFreq) + if minFreq == 0 { + // The infra controller hasn't finished ComputeCPUMinFrequency() yet, so try to + // compute that value now. + var err error + minFreq, err = vs.computeCPUMinFrequency(ctx) + if err != nil { + // minFreq may be non-zero in case of partial success. + return minFreq, err + } + + // Update value if not updated already. + atomic.CompareAndSwapUint64(&vs.minCPUFreq, 0, minFreq) + } + + return minFreq, nil +} + +func (vs *vSphereVMProvider) ComputeCPUMinFrequency(ctx goctx.Context) error { + minFreq, err := vs.computeCPUMinFrequency(ctx) + if err != nil { + // Might have a partial success (non-zero freq): store that if we haven't updated + // the min freq yet, and let the controller retry. This whole min CPU freq thing + // is kind of unfortunate & busted. + atomic.CompareAndSwapUint64(&vs.minCPUFreq, 0, minFreq) + return err + } + + atomic.StoreUint64(&vs.minCPUFreq, minFreq) + return nil +} + +func (vs *vSphereVMProvider) computeCPUMinFrequency(ctx goctx.Context) (uint64, error) { + // Get all the availability zones in order to calculate the minimum + // CPU frequencies for each of the zones' vSphere clusters. + availabilityZones, err := topology.GetAvailabilityZones(ctx, vs.k8sClient) + if err != nil { + return 0, err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return 0, err + } + + if !lib.IsWcpFaultDomainsFSSEnabled() { + ccr, err := vcenter.GetResourcePoolOwnerMoRef(ctx, client.VimClient(), client.Config().ResourcePool) + if err != nil { + return 0, err + } + + // Only expect 1 AZ in this case. + for i := range availabilityZones { + availabilityZones[i].Spec.ClusterComputeResourceMoIDs = []string{ccr.Value} + } + } + + var errs []error + + var minFreq uint64 + for _, az := range availabilityZones { + moIDs := az.Spec.ClusterComputeResourceMoIDs + if len(moIDs) == 0 { + moIDs = []string{az.Spec.ClusterComputeResourceMoId} // HA TEMP + } + + for _, moID := range moIDs { + ccr := object.NewClusterComputeResource(client.VimClient(), + types.ManagedObjectReference{Type: "ClusterComputeResource", Value: moID}) + + freq, err := vcenter.ClusterMinCPUFreq(ctx, ccr) + if err != nil { + errs = append(errs, err) + } else if minFreq == 0 || freq < minFreq { + minFreq = freq + } + } + } + + return minFreq, k8serrors.NewAggregate(errs) +} + +func (vs *vSphereVMProvider) GetTasksByActID(ctx goctx.Context, actID string) (_ []types.TaskInfo, retErr error) { + vcClient, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + taskManager := task.NewManager(vcClient.VimClient()) + filterSpec := types.TaskFilterSpec{ + ActivationId: []string{actID}, + } + + collector, err := taskManager.CreateCollectorForTasks(ctx, filterSpec) + if err != nil { + return nil, errors.Wrapf(err, "failed to create collector for tasks") + } + defer func() { + err = collector.Destroy(ctx) + if retErr == nil { + retErr = err + } + }() + + taskList := make([]types.TaskInfo, 0) + for { + nextTasks, err := collector.ReadNextTasks(ctx, taskHistoryCollectorPageSize) + if err != nil { + log.Error(err, "failed to read next tasks") + return nil, err + } + if len(nextTasks) == 0 { + break + } + taskList = append(taskList, nextTasks...) + } + + log.V(5).Info("found tasks", "actID", actID, "tasks", taskList) + return taskList, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go new file mode 100644 index 000000000..aae8db2f4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go @@ -0,0 +1,261 @@ +// Copyright (c) 2020-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + "context" + "fmt" + + vimtypes "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +// IsVirtualMachineSetResourcePolicyReady checks if the VirtualMachineSetResourcePolicy for the AZ is ready. +func (vs *vSphereVMProvider) IsVirtualMachineSetResourcePolicyReady( + ctx context.Context, + azName string, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (bool, error) { + + client, err := vs.getVcClient(ctx) + if err != nil { + return false, err + } + + folderMoID, rpMoID, err := topology.GetNamespaceFolderAndRPMoID(ctx, vs.k8sClient, azName, resourcePolicy.Namespace) + if err != nil { + return false, err + } + + folderExists, err := vcenter.DoesChildFolderExist(ctx, client.VimClient(), folderMoID, resourcePolicy.Spec.Folder) + if err != nil { + return false, err + } + + rpExists, err := vcenter.DoesChildResourcePoolExist(ctx, client.VimClient(), rpMoID, resourcePolicy.Spec.ResourcePool.Name) + if err != nil { + return false, err + } + + clusterRef, err := vcenter.GetResourcePoolOwnerMoRef(ctx, client.VimClient(), rpMoID) + if err != nil { + return false, err + } + + modulesExist, err := vs.doClusterModulesExist(ctx, client.ClusterModuleClient(), clusterRef.Reference(), resourcePolicy) + if err != nil { + return false, err + } + + if !rpExists || !folderExists || !modulesExist { + log.V(4).Info("Resource policy is not ready", "resourcePolicy", resourcePolicy.Name, + "namespace", resourcePolicy.Name, "az", azName, "resourcePool", rpExists, "folder", folderExists, "modules", modulesExist) + return false, nil + } + + return true, nil +} + +// CreateOrUpdateVirtualMachineSetResourcePolicy creates if a VirtualMachineSetResourcePolicy doesn't exist, updates otherwise. +func (vs *vSphereVMProvider) CreateOrUpdateVirtualMachineSetResourcePolicy( + ctx context.Context, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + folderMoID, rpMoIDs, err := vs.getNamespaceFolderAndRPMoIDs(ctx, resourcePolicy.Namespace) + if err != nil { + return err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + vimClient := client.VimClient() + var errs []error + + _, err = vcenter.CreateFolder(ctx, vimClient, folderMoID, resourcePolicy.Spec.Folder) + if err != nil { + errs = append(errs, err) + } + + for _, rpMoID := range rpMoIDs { + _, err := vcenter.CreateOrUpdateChildResourcePool(ctx, vimClient, rpMoID, &resourcePolicy.Spec.ResourcePool) + if err != nil { + errs = append(errs, err) + } + + clusterRef, err := vcenter.GetResourcePoolOwnerMoRef(ctx, vimClient, rpMoID) + if err == nil { + err = vs.createClusterModules(ctx, client.ClusterModuleClient(), clusterRef.Reference(), resourcePolicy) + } + if err != nil { + errs = append(errs, err) + } + } + + return k8serrors.NewAggregate(errs) +} + +// DeleteVirtualMachineSetResourcePolicy deletes the VirtualMachineSetPolicy. +func (vs *vSphereVMProvider) DeleteVirtualMachineSetResourcePolicy( + ctx context.Context, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + folderMoID, rpMoIDs, err := vs.getNamespaceFolderAndRPMoIDs(ctx, resourcePolicy.Namespace) + if err != nil { + return err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + vimClient := client.VimClient() + var errs []error + + for _, rpMoID := range rpMoIDs { + err := vcenter.DeleteChildResourcePool(ctx, vimClient, rpMoID, resourcePolicy.Spec.ResourcePool.Name) + if err != nil { + errs = append(errs, err) + } + } + + errs = append(errs, vs.deleteClusterModules(ctx, client.ClusterModuleClient(), resourcePolicy)...) + + if err := vcenter.DeleteChildFolder(ctx, vimClient, folderMoID, resourcePolicy.Spec.Folder); err != nil { + errs = append(errs, err) + } + + return k8serrors.NewAggregate(errs) +} + +// doClusterModulesExist checks whether all the ClusterModules for the given VirtualMachineSetResourcePolicy +// have been created and exist in VC for the Session's Cluster. +func (vs *vSphereVMProvider) doClusterModulesExist( + ctx context.Context, + clusterModProvider clustermodules.Provider, + clusterRef vimtypes.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (bool, error) { + + for _, groupName := range resourcePolicy.Spec.ClusterModuleGroups { + _, moduleID := clustermodules.FindClusterModuleUUID(groupName, clusterRef, resourcePolicy) + if moduleID == "" { + return false, nil + } + + exists, err := clusterModProvider.DoesModuleExist(ctx, moduleID, clusterRef) + if !exists || err != nil { + return false, err + } + } + + return true, nil +} + +// createClusterModules creates all the ClusterModules that has not created yet for a +// given VirtualMachineSetResourcePolicy in VC. +func (vs *vSphereVMProvider) createClusterModules( + ctx context.Context, + clusterModProvider clustermodules.Provider, + clusterRef vimtypes.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + var errs []error + + // There is no way to give a name when creating a VC cluster module, so we have to + // resort to using the status as the source of truth. This can result in orphaned + // modules if, for instance, we fail to update the resource policy k8s object. + for _, groupName := range resourcePolicy.Spec.ClusterModuleGroups { + idx, moduleID := clustermodules.FindClusterModuleUUID(groupName, clusterRef, resourcePolicy) + + if moduleID != "" { + // Verify this cluster module exists on VC for this cluster. + exists, err := clusterModProvider.DoesModuleExist(ctx, moduleID, clusterRef) + if err != nil { + errs = append(errs, err) + continue + } + if !exists { + // Status entry is stale. Create below. + moduleID = "" + } + } else { + var err error + // See if there is already a module for this cluster but without the ClusterMoID field + // set that we can claim. + idx, moduleID, err = clustermodules.ClaimClusterModuleUUID(ctx, clusterModProvider, + groupName, clusterRef, resourcePolicy) + if err != nil { + errs = append(errs, err) + continue + } + } + + if moduleID == "" { + var err error + moduleID, err = clusterModProvider.CreateModule(ctx, clusterRef) + if err != nil { + errs = append(errs, err) + continue + } + } + + if idx >= 0 { + resourcePolicy.Status.ClusterModules[idx].ModuleUuid = moduleID + resourcePolicy.Status.ClusterModules[idx].ClusterMoID = clusterRef.Value + } else { + status := vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName, + ModuleUuid: moduleID, + ClusterMoID: clusterRef.Value, + } + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, status) + } + } + + return k8serrors.NewAggregate(errs) +} + +// deleteClusterModules deletes all the ClusterModules associated with a given VirtualMachineSetResourcePolicy in VC. +func (vs *vSphereVMProvider) deleteClusterModules( + ctx context.Context, + clusterModProvider clustermodules.Provider, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) []error { + + var errModStatus []vmopv1.VSphereClusterModuleStatus + var errs []error + + for _, moduleStatus := range resourcePolicy.Status.ClusterModules { + err := clusterModProvider.DeleteModule(ctx, moduleStatus.ModuleUuid) + if err != nil { + errModStatus = append(errModStatus, moduleStatus) + errs = append(errs, err) + } + } + + resourcePolicy.Status.ClusterModules = errModStatus + return errs +} + +func (vs *vSphereVMProvider) getNamespaceFolderAndRPMoIDs( + ctx context.Context, + namespace string) (string, []string, error) { + + folderMoID, rpMoIDs, err := topology.GetNamespaceFolderAndRPMoIDs(ctx, vs.k8sClient, namespace) + if err != nil { + return "", nil, err + } + + if folderMoID == "" { + return "", nil, fmt.Errorf("namespace %s not present in any AvailabilityZones", namespace) + } + + return folderMoID, rpMoIDs, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go new file mode 100644 index 000000000..03c76f704 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go @@ -0,0 +1,234 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "fmt" + "path" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func getVirtualMachineSetResourcePolicy(name, namespace string) *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-resourcepolicy", name), + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: fmt.Sprintf("%s-resourcepool", name), + Reservations: vmopv1.VirtualMachineResourceSpec{}, + Limits: vmopv1.VirtualMachineResourceSpec{}, + }, + Folder: fmt.Sprintf("%s-folder", name), + ClusterModuleGroups: []string{"ControlPlane", "NodeGroup1"}, + }, + } +} + +func resourcePolicyTests() { + Describe("VirtualMachineSetResourcePolicy Tests", func() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + testConfig builder.VCSimTestConfig + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("VirtualMachineSetResourcePolicy", func() { + var ( + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + JustBeforeEach(func() { + testPolicyName := "test-policy" + + resourcePolicy = getVirtualMachineSetResourcePolicy(testPolicyName, nsInfo.Namespace) + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + }) + + JustAfterEach(func() { + Expect(vmProvider.DeleteVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).Should(BeEmpty()) + }) + + It("creates expected cluster modules", func() { + modules := resourcePolicy.Status.ClusterModules + Expect(modules).Should(HaveLen(2)) + module := modules[0] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[0])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + module = modules[1] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[1])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + }) + + Context("for an existing resource policy", func() { + It("should keep existing cluster modules", func() { + status := resourcePolicy.Status.DeepCopy() + + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).To(ContainElements(status.ClusterModules)) + }) + + It("successfully able to find the resource policy", func() { + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, "", resourcePolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + + Context("for an absent resource policy", func() { + It("should fail to find the resource policy without any errors", func() { + failResPolicy := getVirtualMachineSetResourcePolicy("test-policy", nsInfo.Namespace) + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, "", failResPolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("for a resource policy with invalid cluster module", func() { + It("successfully able to delete the resource policy", func() { + resourcePolicy.Status.ClusterModules = append([]vmopv1.VSphereClusterModuleStatus{ + { + GroupName: "invalid-group", + ModuleUuid: "invalid-uuid", + }, + }, resourcePolicy.Status.ClusterModules...) + }) + }) + + It("creates expected resource pool", func() { + rp, err := ctx.GetSingleClusterCompute().ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Make trip through the Finder to populate InventoryPath. + objRef, err := ctx.Finder.ObjectReference(ctx, rp.Reference()) + Expect(err).ToNot(HaveOccurred()) + rp, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + + inventoryPath := path.Join(rp.InventoryPath, nsInfo.Namespace, resourcePolicy.Spec.ResourcePool.Name) + _, err = ctx.Finder.ResourcePool(ctx, inventoryPath) + Expect(err).ToNot(HaveOccurred()) + }) + + It("creates expected child folder", func() { + _, err := ctx.Finder.Folder(ctx, path.Join(nsInfo.Folder.InventoryPath, resourcePolicy.Spec.Folder)) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("when HA is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + It("creates expected cluster modules for each cluster", func() { + moduleCount := len(resourcePolicy.Spec.ClusterModuleGroups) + Expect(moduleCount).To(Equal(2)) + + modules := resourcePolicy.Status.ClusterModules + zoneNames := ctx.ZoneNames + Expect(modules).To(HaveLen(len(zoneNames) * ctx.ClustersPerZone * moduleCount)) + + for zoneIdx, zoneName := range zoneNames { + // NOTE: This assumes some ordering but is the easiest way to test. + moduleIdx := zoneIdx * ctx.ClustersPerZone * moduleCount + modules := modules[moduleIdx : moduleIdx+ctx.ClustersPerZone*moduleCount] + + ccrs := ctx.GetAZClusterComputes(zoneName) + Expect(ccrs).To(HaveLen(ctx.ClustersPerZone)) + + for _, cluster := range ccrs { + clusterMoID := cluster.Reference().Value + + module := modules[0] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[0])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + Expect(module.ClusterMoID).To(Equal(clusterMoID)) + + module = modules[1] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[1])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + Expect(module.ClusterMoID).To(Equal(clusterMoID)) + } + } + }) + + It("should claim cluster module without ClusterMoID set", func() { + Expect(resourcePolicy.Spec.ClusterModuleGroups).ToNot(BeEmpty()) + groupName := resourcePolicy.Spec.ClusterModuleGroups[0] + + moduleStatus := resourcePolicy.Status.DeepCopy() + Expect(moduleStatus.ClusterModules).ToNot(BeEmpty()) + + for i := range resourcePolicy.Status.ClusterModules { + if resourcePolicy.Status.ClusterModules[i].GroupName == groupName { + resourcePolicy.Status.ClusterModules[i].ClusterMoID = "" + } + } + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).To(Equal(moduleStatus.ClusterModules)) + }) + + It("successfully able to find the resource policy in each zone", func() { + for _, zoneName := range ctx.ZoneNames { + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, zoneName, resourcePolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + } + }) + + It("creates expected resource pool for each cluster", func() { + for _, zoneName := range ctx.ZoneNames { + for _, cluster := range ctx.GetAZClusterComputes(zoneName) { + rp, err := cluster.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Make trip through the Finder to populate InventoryPath. + objRef, err := ctx.Finder.ObjectReference(ctx, rp.Reference()) + Expect(err).ToNot(HaveOccurred()) + rp, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + + inventoryPath := path.Join(rp.InventoryPath, nsInfo.Namespace, resourcePolicy.Spec.ResourcePool.Name) + _, err = ctx.Finder.ResourcePool(ctx, inventoryPath) + Expect(err).ToNot(HaveOccurred()) + } + } + }) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_test.go new file mode 100644 index 000000000..5a447c69d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "sync" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func cpuFreqTests() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterface + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + vmProvider = nil + }) + + Context("ComputeCPUMinFrequency", func() { + It("returns success", func() { + Expect(vmProvider.ComputeCPUMinFrequency(ctx)).To(Succeed()) + }) + }) +} + +func initOvfCacheAndLockPoolTests() { + + var ( + expireAfter = 3 * time.Second + checkExpireInterval = 1 * time.Second + maxItems = 3 + + ovfCache *util.Cache[vsphere.VersionedOVFEnvelope] + ovfLockPool *util.LockPool[string, *sync.RWMutex] + ) + + BeforeEach(func() { + ovfCache, ovfLockPool = vsphere.InitOvfCacheAndLockPool( + expireAfter, checkExpireInterval, maxItems) + }) + + AfterEach(func() { + ovfCache = nil + ovfLockPool = nil + }) + + Context("InitOvfCacheAndLockPool", func() { + It("should clean up lock pool when the item is expired in cache", func() { + Expect(ovfCache).ToNot(BeNil()) + Expect(ovfLockPool).ToNot(BeNil()) + + itemID := "test-item-id" + res := ovfCache.Put(itemID, vsphere.VersionedOVFEnvelope{}) + Expect(res).To(Equal(util.CachePutResultCreate)) + curItemLock := ovfLockPool.Get(itemID) + Expect(curItemLock).ToNot(BeNil()) + + Eventually(func() bool { + // ovfLockPool.Get() returns a new lock if the item key is not found. + // So the lock should be different when the item is expired and deleted from pool. + return ovfLockPool.Get(itemID) != curItemLock + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go new file mode 100644 index 000000000..fb7721812 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -0,0 +1,1169 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + goctx "context" + "fmt" + "strings" + "sync" + "text/template" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/placement" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/storage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +// VMCreateArgs contains the arguments needed to create a VM on VC. +type VMCreateArgs struct { + vmlifecycle.CreateArgs + vmlifecycle.BootstrapData + + VMClass *vmopv1.VirtualMachineClass + ResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ImageObj ctrlclient.Object + ImageSpec *vmopv1.VirtualMachineImageSpec + ImageStatus *vmopv1.VirtualMachineImageStatus + + StorageClassesToIDs map[string]string + HasInstanceStorage bool + ChildResourcePoolName string + ChildFolderName string + ClusterMoRef types.ManagedObjectReference + + NetworkResults network.NetworkInterfaceResults +} + +// TODO: Until we sort out what the Session becomes. +type vmUpdateArgs = session.VMUpdateArgs + +const ( + FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" +) + +var ( + createCountLock sync.Mutex + concurrentCreateCount int + + // SkipVMImageCLProviderCheck skips the checks that a VM Image has a Content Library item provider + // since a VirtualMachineImage created for a VM template won't have either. This has been broken for + // a long time but was otherwise masked on how the tests used to be organized. + SkipVMImageCLProviderCheck = false +) + +func (vs *vSphereVMProvider) CreateOrUpdateVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) error { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "createOrUpdateVM")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return err + } + + vcVM, err := vs.getVM(vmCtx, client, false) + if err != nil { + return err + } + + if vcVM == nil { + var createArgs *VMCreateArgs + + vcVM, createArgs, err = vs.createVirtualMachine(vmCtx, client) + if err != nil { + return err + } + + if vcVM == nil { + // Creation was not ready or blocked for some reason. We depend on the controller + // to eventually retry the create. + return nil + } + + return vs.createdVirtualMachineFallthroughUpdate(vmCtx, vcVM, client, createArgs) + } + + return vs.updateVirtualMachine(vmCtx, vcVM, client, nil) +} + +func (vs *vSphereVMProvider) DeleteVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) error { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "deleteVM")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return err + } + + vcVM, err := vs.getVM(vmCtx, client, false) + if err != nil { + return err + } else if vcVM == nil { + // VM does not exist. + return nil + } + + return virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) +} + +func (vs *vSphereVMProvider) PublishVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine, + vmPub *vmopv1.VirtualMachinePublishRequest, + cl *imgregv1a1.ContentLibrary, + actID string) (string, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: ctx, + // Update logger info + Logger: log.WithValues("vmName", vm.NamespacedName()). + WithValues("clName", fmt.Sprintf("%s/%s", cl.Namespace, cl.Name)). + WithValues("vmPubName", fmt.Sprintf("%s/%s", vmPub.Namespace, vmPub.Name)), + VM: vm, + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return "", errors.Wrapf(err, "failed to get vCenter client") + } + + itemID, err := virtualmachine.CreateOVF(vmCtx, client.RestClient(), vmPub, cl, actID) + if err != nil { + return "", err + } + + return itemID, nil +} + +// BackupVirtualMachine backs up the VM data required for restore. +func (vs *vSphereVMProvider) BackupVirtualMachine(ctx goctx.Context, vm *vmopv1.VirtualMachine) error { + // TODO + return nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineGuestHeartbeat( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "heartbeat")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return "", err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return "", err + } + + status, err := virtualmachine.GetGuestHeartBeatStatus(vmCtx, vcVM) + if err != nil { + return "", err + } + + return status, nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineWebMKSTicket( + ctx goctx.Context, + vm *vmopv1.VirtualMachine, + pubKey string) (string, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "webconsole")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return "", err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return "", err + } + + ticket, err := virtualmachine.GetWebConsoleTicket(vmCtx, vcVM, pubKey) + if err != nil { + return "", err + } + + return ticket, nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineHardwareVersion( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) (int32, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "hardware-version")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return 0, err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return 0, err + } + + var o mo.VirtualMachine + err = vcVM.Properties(vmCtx, vcVM.Reference(), []string{"config.version"}, &o) + if err != nil { + return 0, err + } + + return contentlibrary.ParseVirtualHardwareVersion(o.Config.Version), nil +} + +func (vs *vSphereVMProvider) createVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*object.VirtualMachine, *VMCreateArgs, error) { + + createArgs, err := vs.vmCreateGetArgs(vmCtx, vcClient) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateDoPlacement(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateGetFolderAndRPMoIDs(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateFixupConfigSpec(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateIsReady(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + // BMV: This is about where we used to do this check but it prb make more sense to do + // earlier, as to limit wasted work. Before DoPlacement() is likely the best place so + // the window between the placement decision and creating the VM on VC is small(ish). + allowed, createDeferFn, err := vs.vmCreateConcurrentAllowed(vmCtx) + if err != nil { + return nil, nil, err + } else if !allowed { + return nil, nil, nil + } + defer createDeferFn() + + moRef, err := vmlifecycle.CreateVirtualMachine( + vmCtx, + vcClient.ContentLibClient(), + vcClient.RestClient(), + vcClient.Finder(), + &createArgs.CreateArgs) + if err != nil { + vmCtx.Logger.Error(err, "CreateVirtualMachine failed") + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionCreated, "Error", err.Error()) + return nil, nil, err + } + + vmCtx.VM.Status.UniqueID = moRef.Reference().Value + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionCreated) + + return object.NewVirtualMachine(vcClient.VimClient(), *moRef), createArgs, nil +} + +func (vs *vSphereVMProvider) createdVirtualMachineFallthroughUpdate( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + // TODO: In the common case, we'll call directly into update right after create succeeds, and + // can use the createArgs to avoid doing a bunch of lookup work again. + + return vs.updateVirtualMachine(vmCtx, vcVM, vcClient, createArgs) +} + +func (vs *vSphereVMProvider) updateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + vmCtx.Logger.V(4).Info("Updating VirtualMachine") + + { + // Hack - create just enough of the Session that's needed for update + + cluster, err := virtualmachine.GetVMClusterComputeResource(vmCtx, vcVM) + if err != nil { + return err + } + + ses := &session.Session{ + K8sClient: vs.k8sClient, + Client: vcClient, + Finder: vcClient.Finder(), + Cluster: cluster, + } + + getUpdateArgsFn := func() (*vmUpdateArgs, error) { + // TODO: Use createArgs if we already got them + _ = createArgs + return vs.vmUpdateGetArgs(vmCtx) + } + + err = ses.UpdateVirtualMachine(vmCtx, vcVM, getUpdateArgsFn) + if err != nil { + return err + } + } + + return nil +} + +// vmCreateDoPlacement determines placement of the VM prior to creating the VM on VC. +func (vs *vSphereVMProvider) vmCreateDoPlacement( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + placementConfigSpec := virtualmachine.CreateConfigSpecForPlacement( + vmCtx, + createArgs.ConfigSpec, + createArgs.StorageClassesToIDs) + + result, err := placement.Placement( + vmCtx, + vs.k8sClient, + vcClient.VimClient(), + placementConfigSpec, + createArgs.ChildResourcePoolName) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady, "NotReady", err.Error()) + return err + } + + if result.PoolMoRef.Value != "" { + createArgs.ResourcePoolMoID = result.PoolMoRef.Value + } + + if result.HostMoRef != nil { + createArgs.HostMoID = result.HostMoRef.Value + } + + if result.InstanceStoragePlacement { + hostMoID := createArgs.HostMoID + + if hostMoID == "" { + return fmt.Errorf("placement result missing host required for instance storage") + } + + hostFQDN, err := vcenter.GetESXHostFQDN(vmCtx, vcClient.VimClient(), hostMoID) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady, "NotReady", err.Error()) + return err + } + + if vmCtx.VM.Annotations == nil { + vmCtx.VM.Annotations = map[string]string{} + } + vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey] = hostMoID + vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeAnnotationKey] = hostFQDN + } + + if result.ZonePlacement { + if vmCtx.VM.Labels == nil { + vmCtx.VM.Labels = map[string]string{} + } + // Note if the VM create fails for some reason, but this label gets updated on the k8s VM, + // then this is the pre-assigned zone on later create attempts. + vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey] = result.ZoneName + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady) + + return nil +} + +// vmCreateGetFolderAndRPMoIDs gets the MoIDs of the Folder and Resource Pool the VM will be created under. +func (vs *vSphereVMProvider) vmCreateGetFolderAndRPMoIDs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if createArgs.ResourcePoolMoID == "" { + // We did not do placement so find this namespace/zone ResourcePool and Folder. + + nsFolderMoID, rpMoID, err := topology.GetNamespaceFolderAndRPMoID(vmCtx, vs.k8sClient, + vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey], vmCtx.VM.Namespace) + if err != nil { + return err + } + + // If this VM has a ResourcePolicy ResourcePool, lookup the child ResourcePool under the + // namespace/zone's root ResourcePool. This will be the VM's ResourcePool. + if createArgs.ChildResourcePoolName != "" { + parentRP := object.NewResourcePool(vcClient.VimClient(), + types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + childRP, err := vcenter.GetChildResourcePool(vmCtx, parentRP, createArgs.ChildResourcePoolName) + if err != nil { + return err + } + + rpMoID = childRP.Reference().Value + } + + createArgs.ResourcePoolMoID = rpMoID + createArgs.FolderMoID = nsFolderMoID + + } else { + // Placement already selected the ResourcePool/Cluster, so we just need this namespace's Folder. + nsFolderMoID, err := topology.GetNamespaceFolderMoID(vmCtx, vs.k8sClient, vmCtx.VM.Namespace) + if err != nil { + return err + } + + createArgs.FolderMoID = nsFolderMoID + } + + // If this VM has a ResourcePolicy Folder, lookup the child Folder under the namespace's Folder. + // This will be the VM's parent Folder in the VC inventory. + if createArgs.ChildFolderName != "" { + parentFolder := object.NewFolder(vcClient.VimClient(), + types.ManagedObjectReference{Type: "Folder", Value: createArgs.FolderMoID}) + + childFolder, err := vcenter.GetChildFolder(vmCtx, parentFolder, createArgs.ChildFolderName) + if err != nil { + return err + } + + createArgs.FolderMoID = childFolder.Reference().Value + } + + // Now that we know the ResourcePool, use that to look up the CCR. + clusterMoRef, err := vcenter.GetResourcePoolOwnerMoRef(vmCtx, vcClient.VimClient(), createArgs.ResourcePoolMoID) + if err != nil { + return err + } + createArgs.ClusterMoRef = clusterMoRef + + return nil +} + +func (vs *vSphereVMProvider) vmCreateFixupConfigSpec( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + fixedUp, err := network.ResolveBackingPostPlacement( + vmCtx, + vcClient.VimClient(), + createArgs.ClusterMoRef, + &createArgs.NetworkResults) + if err != nil { + return err + } + + if fixedUp { + // Now that the backing is resolved for this CCR, re-zip to update the ConfigSpec. What a mess. + err = vs.vmCreateGenConfigSpecZipNetworkInterfaces(vmCtx, createArgs) + if err != nil { + return err + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateIsReady( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if policy := createArgs.ResourcePolicy; policy != nil { + // TODO: May want to do this as to filter the placement candidates. + exists, err := vs.doClusterModulesExist(vmCtx, vcClient.ClusterModuleClient(), createArgs.ClusterMoRef, policy) + if err != nil { + return err + } else if !exists { + return fmt.Errorf("VirtualMachineSetResourcePolicy cluster module is not ready") + } + } + + if createArgs.HasInstanceStorage { + if _, ok := vmCtx.VM.Annotations[constants.InstanceStoragePVCsBoundAnnotationKey]; !ok { + return fmt.Errorf("instance storage PVCs are not bound yet") + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateConcurrentAllowed(vmCtx context.VirtualMachineContextA2) (bool, func(), error) { + maxDeployThreads, ok := vmCtx.Value(context.MaxDeployThreadsContextKey).(int) + if !ok { + return false, nil, fmt.Errorf("MaxDeployThreadsContextKey missing from context") + } + + createCountLock.Lock() + if concurrentCreateCount >= maxDeployThreads { + createCountLock.Unlock() + vmCtx.Logger.Info("Too many create VirtualMachine already occurring. Re-queueing request") + return false, nil, nil + } + + concurrentCreateCount++ + createCountLock.Unlock() + + decrementFn := func() { + createCountLock.Lock() + concurrentCreateCount-- + createCountLock.Unlock() + } + + return true, decrementFn, nil +} + +func (vs *vSphereVMProvider) vmCreateGetArgs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*VMCreateArgs, error) { + + createArgs, err := vs.vmCreateGetPrereqs(vmCtx, vcClient) + if err != nil { + return nil, err + } + + err = vs.vmCreateDoNetworking(vmCtx, vcClient, createArgs) + if err != nil { + return nil, err + } + + err = vs.vmCreateGenConfigSpec(vmCtx, createArgs) + if err != nil { + return nil, err + } + + err = vs.vmCreateValidateArgs(vmCtx, vcClient, createArgs) + if err != nil { + return nil, err + } + + return createArgs, nil +} + +// vmCreateGetPrereqs returns the VMCreateArgs populated with the k8s objects required to +// create the VM on VC. +func (vs *vSphereVMProvider) vmCreateGetPrereqs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*VMCreateArgs, error) { + + createArgs := &VMCreateArgs{} + var prereqErrs []error + + if err := vs.vmCreateGetVirtualMachineClass(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetVirtualMachineImage(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetSetResourcePolicy(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetBootstrap(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetStoragePrereqs(vmCtx, vcClient, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + // This is about the point where historically we'd declare the prereqs ready or not. There + // is still a lot of work to do - and things to fail - before the actual create, but there + // is no point in continuing if the above checks aren't met since we are missing data + // required to create the VM. + if len(prereqErrs) > 0 { + return nil, k8serrors.NewAggregate(prereqErrs) + } + + // Note that once the VM is created, it is hard for us to later resolve what image was used, + // since a NS or cluster scoped image could have been created or deleted. + vmCtx.VM.Status.Image = &common.LocalObjectRef{ + APIVersion: createArgs.ImageObj.GetObjectKind().GroupVersionKind().Version, + Kind: createArgs.ImageObj.GetObjectKind().GroupVersionKind().Kind, + Name: createArgs.ImageObj.GetName(), + } + + vmCtx.VM.Status.Class = &common.LocalObjectRef{ + APIVersion: createArgs.VMClass.APIVersion, + Kind: createArgs.VMClass.Kind, + Name: createArgs.VMClass.Name, + } + + return createArgs, nil +} + +func (vs *vSphereVMProvider) vmCreateGetVirtualMachineClass( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + vmClass, err := GetVirtualMachineClass(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.VMClass = vmClass + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetVirtualMachineImage( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + imageObj, imageSpec, imageStatus, err := GetVirtualMachineImageSpecAndStatus(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.ImageObj = imageObj + createArgs.ImageSpec = imageSpec + createArgs.ImageStatus = imageStatus + + // This is clunky, but we need to know how to use the image to create the VM. Our only supported + // method is via the ContentLibrary, so check if this image was derived from a CL item. + switch imageSpec.ProviderRef.Kind { + case "ClusterContentLibraryItem", "ContentLibraryItem": + createArgs.UseContentLibrary = true + createArgs.ProviderItemID = imageStatus.ProviderItemID + default: + if !SkipVMImageCLProviderCheck { + err := fmt.Errorf("unsupported image provider kind: %s", imageSpec.ProviderRef.Kind) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, "NotSupported", err.Error()) + return err + } + // Testing only: we'll clone the source VM found in the Inventory. + createArgs.UseContentLibrary = false + createArgs.ProviderItemID = vmCtx.VM.Spec.ImageName + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetSetResourcePolicy( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + resourcePolicy, err := GetVMSetResourcePolicy(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + // The SetResourcePolicy is optional (TKG VMs will always have it). + if resourcePolicy != nil { + createArgs.ResourcePolicy = resourcePolicy + createArgs.ChildFolderName = resourcePolicy.Spec.Folder + createArgs.ChildResourcePoolName = resourcePolicy.Spec.ResourcePool.Name + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetBootstrap( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + data, vAppData, vAppExData, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.BootstrapData.Data = data + createArgs.BootstrapData.VAppData = vAppData + createArgs.BootstrapData.VAppExData = vAppExData + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetStoragePrereqs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if lib.IsInstanceStorageFSSEnabled() { + // To determine all the storage profiles, we need the class because of the possibility of + // InstanceStorage volumes. If we weren't able to get the class earlier, still check & set + // the storage condition because instance storage usage is rare, it is helpful to report + // as many prereqs as possible, and we'll reevaluate this once the class is available. + if createArgs.VMClass != nil { + // Add the class's instance storage disks - if any - to the VM.Spec. Once the instance + // storage disks are added to the VM, they are set in stone even if the class itself or + // the VM's assigned class changes. + createArgs.HasInstanceStorage = AddInstanceStorageVolumes(vmCtx, createArgs.VMClass) + } + } + + storageClassesToIDs, err := storage.GetVMStoragePoliciesIDs(vmCtx, vs.k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady, reason, msg) + return err + } + + vmStorageProfileID := storageClassesToIDs[vmCtx.VM.Spec.StorageClass] + + provisioningType, err := virtualmachine.GetDefaultDiskProvisioningType(vmCtx, vcClient, vmStorageProfileID) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady, reason, msg) + return err + } + + createArgs.StorageClassesToIDs = storageClassesToIDs + createArgs.StorageProvisioning = provisioningType + createArgs.StorageProfileID = vmStorageProfileID + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateDoNetworking( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + networkSpec := &vmCtx.VM.Spec.Network + if networkSpec.Disabled { + // No connected networking for this VM. Any EthCards will be removed later. + return nil + } + + interfaces := networkSpec.Interfaces + if len(interfaces) == 0 { + // VM gets one automatic NIC. Create the default interface from fields in the network spec. + defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: networkSpec.DeviceName, + Addresses: networkSpec.Addresses, + DHCP4: networkSpec.DHCP4, + DHCP6: networkSpec.DHCP6, + Gateway4: networkSpec.Gateway4, + Gateway6: networkSpec.Gateway6, + MTU: networkSpec.MTU, + Nameservers: networkSpec.Nameservers, + Routes: networkSpec.Routes, + SearchDomains: networkSpec.SearchDomains, + } + + if defaultInterface.Name == "" { + defaultInterface.Name = "eth0" + } + if networkSpec.Network != nil { + defaultInterface.Network = *networkSpec.Network + } + + interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} + } + + results, err := network.CreateAndWaitForNetworkInterfaces( + vmCtx, + vs.k8sClient, + vcClient.VimClient(), + vcClient.Finder(), + nil, // Don't know the CCR yet (needed to resolve backings for NSX-T) + interfaces) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionNetworkReady, "NotReady", err.Error()) + return err + } + + createArgs.NetworkResults = results + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionNetworkReady) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpec( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + // TODO: This is a partial dupe of what's done in the update path in the remaining Session code. I got + // tired of trying to keep that in sync so we get to live with a frankenstein thing longer. + + var vmClassConfigSpec *types.VirtualMachineConfigSpec + if rawConfigSpec := createArgs.VMClass.Spec.ConfigSpec; lib.IsVMClassAsConfigFSSDaynDateEnabled() && len(rawConfigSpec) > 0 { + configSpec, err := GetVMClassConfigSpec(rawConfigSpec) + if err != nil { + return err + } + vmClassConfigSpec = configSpec + } else { + vmClassConfigSpec = virtualmachine.ConfigSpecFromVMClassDevices(&createArgs.VMClass.Spec) + } + + var minCPUFreq uint64 + if res := createArgs.VMClass.Spec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + freq, err := vs.getOrComputeCPUMinFrequency(vmCtx) + if err != nil { + return err + } + minCPUFreq = freq + } + + createArgs.ConfigSpec = virtualmachine.CreateConfigSpec( + vmCtx, + vmClassConfigSpec, + &createArgs.VMClass.Spec, + createArgs.ImageStatus, + minCPUFreq) + + // TODO: This should be in CreateConfigSpec() + if createArgs.ConfigSpec.Version == "" { + imageVer := int32(0) + if createArgs.ImageStatus.HardwareVersion != nil { + imageVer = *createArgs.ImageStatus.HardwareVersion + } + + version := HardwareVersionForPVCandPCIDevices(imageVer, createArgs.ConfigSpec, HasPVC(vmCtx.VM.Spec)) + if version != 0 { + createArgs.ConfigSpec.Version = fmt.Sprintf("vmx-%d", version) + } + } + + err := vs.vmCreateGenConfigSpecExtraConfig(vmCtx, createArgs) + if err != nil { + return err + } + + err = vs.vmCreateGenConfigSpecChangeBootDiskSize(vmCtx, createArgs) + if err != nil { + return err + } + + err = vs.vmCreateGenConfigSpecZipNetworkInterfaces(vmCtx, createArgs) + if err != nil { + return err + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecExtraConfig( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + ecMap := make(map[string]string, len(vs.globalExtraConfig)) + + // The only use of this template is for the JSON_EXTRA_CONFIG that is set in gce2e env + // to populate {{.ImageName }} so vcsim will create a container for the VM. + // BMV: This should be removable now that vcsim gce2e is gone. + renderTemplateFn := func(name, text string) string { + t, err := template.New(name).Parse(text) + if err != nil { + return text + } + b := strings.Builder{} + if err := t.Execute(&b, createArgs.ImageStatus); err != nil { + return text + } + return b.String() + } + for k, v := range vs.globalExtraConfig { + ecMap[k] = renderTemplateFn(k, v) + } + + hasPassthroughDevices := len(util.SelectVirtualPCIPassthrough(util.DevicesFromConfigSpec(createArgs.ConfigSpec))) > 0 + + if hasPassthroughDevices || createArgs.HasInstanceStorage { + ecMap[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + } + + if hasPassthroughDevices { + mmioSize := vmCtx.VM.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] + if mmioSize == "" { + mmioSize = constants.PCIPassthruMMIOSizeDefault + } + if mmioSize != "0" { + ecMap[constants.PCIPassthruMMIOExtraConfigKey] = constants.ExtraConfigTrue + ecMap[constants.PCIPassthruMMIOSizeExtraConfigKey] = mmioSize + } + } + + // The ConfigSpec's current ExtraConfig values (that came from the class) take precedence over what was set here. + createArgs.ConfigSpec.ExtraConfig = util.AppendNewExtraConfigValues(createArgs.ConfigSpec.ExtraConfig, ecMap) + + // Leave constants.VMOperatorV1Alpha1ExtraConfigKey for the update path (if that's still even needed) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecChangeBootDiskSize( + vmCtx context.VirtualMachineContextA2, + _ *VMCreateArgs) error { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil + } + + // TODO: How to we determine the DeviceKey for the DeviceChange entry? We probably have to + // crack the image/source, which is hard to do ATM. Punt on this for a placement consideration + // and we'll resize the boot (first) disk after VM create like before. + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecZipNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + if vmCtx.VM.Spec.Network.Disabled { + util.RemoveDevicesFromConfigSpec(createArgs.ConfigSpec, util.IsEthernetCard) + return nil + } + + resultsIdx := 0 + var unmatchedEthDevices []int + + for idx := range createArgs.ConfigSpec.DeviceChange { + spec := createArgs.ConfigSpec.DeviceChange[idx].GetVirtualDeviceConfigSpec() + if spec == nil || !util.IsEthernetCard(spec.Device) { + continue + } + + device := spec.Device + ethCard := device.(types.BaseVirtualEthernetCard).GetVirtualEthernetCard() + + if resultsIdx < len(createArgs.NetworkResults.Results) { + err := network.ApplyInterfaceResultToVirtualEthCard(vmCtx, ethCard, &createArgs.NetworkResults.Results[resultsIdx]) + if err != nil { + return err + } + resultsIdx++ + + } else { + // This ConfigSpec Ethernet device does not have a corresponding entry in the VM Spec, so we + // won't ever have a backing for it. Remove it from the ConfigSpec since that is the easiest + // thing to do, since extra NICs can cause later complications around GOSC and other customizations. + // The downside with this is that if a NIC is added to the VM Spec, it won't necessarily have this + // config but the default. Revisit this later if we don't like that behavior. + unmatchedEthDevices = append(unmatchedEthDevices, idx-len(unmatchedEthDevices)) + } + } + + if len(unmatchedEthDevices) > 0 { + deviceChange := createArgs.ConfigSpec.DeviceChange + for _, idx := range unmatchedEthDevices { + deviceChange = append(deviceChange[:idx], deviceChange[idx+1:]...) + } + createArgs.ConfigSpec.DeviceChange = deviceChange + } + + // Any remaining VM Spec network interfaces were not matched with a device in the ConfigSpec, so + // create a default virtual ethernet card for them. + for i := resultsIdx; i < len(createArgs.NetworkResults.Results); i++ { + ethCardDev, err := network.CreateDefaultEthCard(vmCtx, &createArgs.NetworkResults.Results[i]) + if err != nil { + return err + } + + // May not have the backing yet (NSX-T). We come back through here after placement once we + // know the backing. + if ethCardDev != nil { + createArgs.ConfigSpec.DeviceChange = append(createArgs.ConfigSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: ethCardDev, + }) + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateValidateArgs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + // Some of this would be better done in the validation webhook but have it here for now. + cfg := vcClient.Config() + + if cfg.StorageClassRequired { + // In WCP this is always required. + if vmCtx.VM.Spec.StorageClass == "" { + return fmt.Errorf("StorageClass is required but not specified") + } + + if createArgs.StorageProfileID == "" { + // GetVMStoragePoliciesIDs() would have returned an error if the policy didn't exist, but + // ensure the field is set. + return fmt.Errorf("no StorageProfile found for StorageClass %s", vmCtx.VM.Spec.StorageClass) + } + + } else if vmCtx.VM.Spec.StorageClass == "" { + // This is only set in gce2e. + if cfg.Datastore == "" { + return fmt.Errorf("no Datastore provided in configuration") + } + + datastore, err := vcClient.Finder().Datastore(vmCtx, cfg.Datastore) + if err != nil { + return fmt.Errorf("failed to find Datastore %s: %w", cfg.Datastore, err) + } + + createArgs.DatastoreMoID = datastore.Reference().Value + } + + return nil +} + +func (vs *vSphereVMProvider) vmUpdateGetArgs( + vmCtx context.VirtualMachineContextA2) (*vmUpdateArgs, error) { + + vmClass, err := GetVirtualMachineClass(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + resourcePolicy, err := GetVMSetResourcePolicy(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + data, vAppData, vAppExData, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + updateArgs := &vmUpdateArgs{} + updateArgs.VMClass = vmClass + updateArgs.ResourcePolicy = resourcePolicy + updateArgs.BootstrapData.Data = data + updateArgs.BootstrapData.VAppData = vAppData + updateArgs.BootstrapData.VAppExData = vAppExData + + if res := vmClass.Spec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + freq, err := vs.getOrComputeCPUMinFrequency(vmCtx) + if err != nil { + return nil, err + } + updateArgs.MinCPUFreq = freq + } + + var vmClassConfigSpec *types.VirtualMachineConfigSpec + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if cs := updateArgs.VMClass.Spec.ConfigSpec; cs != nil { + var err error + vmClassConfigSpec, err = GetVMClassConfigSpec(cs) + if err != nil { + return nil, err + } + } + } + + var vmImageStatus *vmopv1.VirtualMachineImageStatus + // Only get VM image when this is the VM first boot. + if isVMFirstBoot(vmCtx) { + var err error + _, _, vmImageStatus, err = GetVirtualMachineImageSpecAndStatus(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + // The only use of this is for the global JSON_EXTRA_CONFIG to set the image name. + // The global extra config should only be set during first boot. + // TODO: We can just finally kill this with the demise of old gce2e? + renderTemplateFn := func(name, text string) string { + t, err := template.New(name).Parse(text) + if err != nil { + return text + } + b := strings.Builder{} + if err := t.Execute(&b, vmImageStatus); err != nil { + return text + } + return b.String() + } + extraConfig := make(map[string]string, len(vs.globalExtraConfig)) + for k, v := range vs.globalExtraConfig { + extraConfig[k] = renderTemplateFn(k, v) + } + updateArgs.ExtraConfig = extraConfig + + // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot + // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled + // when the vmMetadata is nil or when the transport requested is not CloudInit. + // TODO: Is this still actually needed? + updateArgs.VirtualMachineImageV1Alpha1Compatible = + conditions.IsTrueFromConditions(vmImageStatus.Conditions, "VirtualMachineImageV1Alpha1Compatible" /*vmopv1.VirtualMachineImageV1Alpha1CompatibleCondition*/) + } + + updateArgs.ConfigSpec = virtualmachine.CreateConfigSpec( + vmCtx, + vmClassConfigSpec, + &updateArgs.VMClass.Spec, + vmImageStatus, + updateArgs.MinCPUFreq) + + return updateArgs, nil +} + +func isVMFirstBoot(vmCtx context.VirtualMachineContextA2) bool { + if _, ok := vmCtx.VM.Annotations[FirstBootDoneAnnotation]; ok { + return false + } + + return true +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go new file mode 100644 index 000000000..5b516a6cb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go @@ -0,0 +1,256 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + goctx "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +// vmE2ETests() tries to close the gap in the existing vmTests() have in the sense that we don't do e2e-like +// tests of the typical VM create/update workflow. This somewhat of a super-set of the vmTests() but those +// tests are already kind of unwieldy and in places, and until we switch over to v1a2, I don't +// to disturb that file so keeping things in sync easier. +// For now, these tests focus on a real - VDS or NSX-T - network env w/ cloud init, and we'll see how these +// need to evolve. +func vmE2ETests() { + + var ( + initObjects []client.Object + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + nsInfo builder.WorkloadNamespaceInfo + + vm *vmopv1.VirtualMachine + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + // Speed up tests until we Watch the network types. Sigh. + network.RetryTimeout = 1 * time.Millisecond + + testConfig = builder.VCSimTestConfig{ + WithV1A2: true, + WithContentLibrary: true, + WithVMClassAsConfigDaynDate: true, + } + + vm = builder.DummyBasicVirtualMachineA2("test-vm", "") + vmClass = builder.DummyVirtualMachineClassA2() + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + ctx.Context = goctx.WithValue(ctx.Context, context.MaxDeployThreadsContextKey, 1) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + nsInfo = ctx.CreateWorkloadNamespace() + + vmClass.Namespace = nsInfo.Namespace + Expect(ctx.Client.Create(ctx, vmClass)).To(Succeed()) + vmClass.Status.Ready = true + Expect(ctx.Client.Status().Update(ctx, vmClass)).To(Succeed()) + + cloudInitSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cloud-init-secret", + Namespace: nsInfo.Namespace, + }, + StringData: map[string]string{ + "user-value": "", + }, + } + Expect(ctx.Client.Create(ctx, cloudInitSecret)).To(Succeed()) + + vm.Namespace = nsInfo.Namespace + vm.Spec.ClassName = vmClass.Name + vm.Spec.ImageName = ctx.ContentLibraryImageName + vm.Spec.StorageClass = ctx.StorageClassName + vm.Spec.Network.Nameservers = []string{"1.1.1.1", "8.8.8.8"} + vm.Spec.Network.SearchDomains = []string{"vmware.local"} + vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{ + RawCloudConfig: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cloudInitSecret.Name, + }, + }, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + vmProvider = nil + nsInfo = builder.WorkloadNamespaceInfo{} + + vm = nil + vmClass = nil + }) + + Context("VDS", func() { + + const ( + networkName = "my-vds-network" + interfaceName = "eth0" + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvVDS + + vm.Spec.Network.Network = &common.PartialObjectRef{ + Name: networkName, + } + }) + + It("DoIt", func() { + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("simulate successful NetOP reconcile", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.NetworkName).To(Equal(networkName)) + + netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value + netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.110", + IPFamily: netopv1alpha1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + devList, err := vcVM.Device(ctx) + Expect(err).ToNot(HaveOccurred()) + + // For now just check the expected Nic backing. + By("Has expected NIC backing", func() { + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + + dev1 := l[0].GetVirtualDevice() + backingInfo, ok := dev1.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + }) + }) + + Context("NSX-T", func() { + + const ( + networkName = "my-nsxt-network" + interfaceName = "eth0" + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + + vm.Spec.Network.Network = &common.PartialObjectRef{ + Name: networkName, + } + }) + + It("DoIt", func() { + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("simulate successful NCP reconcile", func() { + netInterface := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.VirtualNetwork).To(Equal(networkName)) + + netInterface.Status.MacAddress = "01-23-45-67-89-AB-CD-EF" + netInterface.Status.ProviderStatus = &ncpv1alpha1.VirtualNetworkInterfaceProviderStatus{ + NsxLogicalSwitchID: builder.NsxTLogicalSwitchUUID, + } + netInterface.Status.IPAddresses = []ncpv1alpha1.VirtualNetworkInterfaceIP{ + { + IP: "192.168.1.110", + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + } + netInterface.Status.Conditions = []ncpv1alpha1.VirtualNetworkCondition{ + { + Type: "Ready", + Status: "True", + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + devList, err := vcVM.Device(ctx) + Expect(err).ToNot(HaveOccurred()) + + // For now just check the expected Nic backing. + By("Has expected NIC backing", func() { + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + + dev1 := l[0].GetVirtualDevice() + backingInfo, ok := dev1.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go new file mode 100644 index 000000000..fbf325f43 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go @@ -0,0 +1,1813 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "bytes" + goctx "context" + "encoding/json" + "fmt" + "math/rand" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vapi/cluster" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vmTests() { + + const ( + // Hardcoded vcsim CPU frequency. + vcsimCPUFreq = 2294 + + // Default network created for free by vcsim. + dvpgName = "DC0_DVPG0" + ) + + var ( + initObjects []client.Object + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + nsInfo builder.WorkloadNamespaceInfo + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + ctx.Context = goctx.WithValue(ctx.Context, context.MaxDeployThreadsContextKey, 1) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + vmProvider = nil + nsInfo = builder.WorkloadNamespaceInfo{} + }) + + Context("Create/Update/Delete VirtualMachine", func() { + var ( + vm *vmopv1.VirtualMachine + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + testConfig.WithContentLibrary = true + vmClass = builder.DummyVirtualMachineClassA2() + vm = builder.DummyBasicVirtualMachineA2("test-vm", "") + + // Reduce diff from old tests: by default don't create an NIC. + vm.Spec.Network.Disabled = true + }) + + AfterEach(func() { + vmClass = nil + vm = nil + }) + + JustBeforeEach(func() { + vmClass.Namespace = nsInfo.Namespace + Expect(ctx.Client.Create(ctx, vmClass)).To(Succeed()) + vmClass.Status.Ready = true + Expect(ctx.Client.Status().Update(ctx, vmClass)).To(Succeed()) + + clusterVMImage := &vmopv1.ClusterVirtualMachineImage{} + if testConfig.WithContentLibrary { + Expect(ctx.Client.Get(ctx, client.ObjectKey{Name: ctx.ContentLibraryImageName}, clusterVMImage)).To(Succeed()) + } else { + // BMV: VM creation without CL is broken - and has been for a long while - since we assume + // the VM Image will always point to a ContentLibrary item. + // Hack around that with this knob so we can continue to test the VM clone path. + vsphere.SkipVMImageCLProviderCheck = true + + // Use the default VM created by vcsim as the source. + clusterVMImage = builder.DummyClusterVirtualMachineImageA2("DC0_C0_RP0_VM0") + Expect(ctx.Client.Create(ctx, clusterVMImage)).To(Succeed()) + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) + Expect(ctx.Client.Status().Update(ctx, clusterVMImage)).To(Succeed()) + } + + vm.Namespace = nsInfo.Namespace + vm.Spec.ClassName = vmClass.Name + vm.Spec.ImageName = clusterVMImage.Name + vm.Spec.StorageClass = ctx.StorageClassName + }) + + AfterEach(func() { + vsphere.SkipVMImageCLProviderCheck = false + }) + + createOrUpdateAndGetVcVM := func( + ctx *builder.TestContextForVCSim, + vm *vmopv1.VirtualMachine) (*object.VirtualMachine, error) { + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + if err != nil { + return nil, err + } + + ExpectWithOffset(1, vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + ExpectWithOffset(1, vcVM).ToNot(BeNil()) + return vcVM, nil + } + + Context("VMClassAsConfigDaynDate FSS is enabled", func() { + + var ( + vcVM *object.VirtualMachine + configSpec *types.VirtualMachineConfigSpec + ethCard types.VirtualEthernetCard + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + testConfig.WithVMClassAsConfigDaynDate = true + + ethCard = types.VirtualEthernetCard{ + VirtualDevice: types.VirtualDevice{ + Key: 4000, + DeviceInfo: &types.Description{ + Label: "test-configspec-nic-label", + Summary: "VM Network", + }, + SlotInfo: &types.VirtualDevicePciBusSlotInfo{ + VirtualDeviceBusSlotInfo: types.VirtualDeviceBusSlotInfo{}, + PciSlotNumber: 32, + }, + ControllerKey: 100, + }, + AddressType: string(types.VirtualEthernetCardMacTypeGenerated), + MacAddress: "00:0c:29:93:d7:27", + ResourceAllocation: &types.VirtualEthernetCardResourceAllocation{ + Reservation: pointer.Int64(42), + }, + } + }) + + JustBeforeEach(func() { + if configSpec != nil { + var w bytes.Buffer + enc := types.NewJSONEncoder(&w) + Expect(enc.Encode(configSpec)).To(Succeed()) + + // Update the VM Class with the XML. + vmClass.Spec.ConfigSpec = w.Bytes() + Expect(ctx.Client.Update(ctx, vmClass)).To(Succeed()) + } + + vm.Spec.Network.Disabled = false + vm.Spec.Network.Network = &common.PartialObjectRef{Name: dvpgName} + + var err error + vcVM, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + vcVM = nil + configSpec = nil + }) + + Context("VM Class has no ConfigSpec", func() { + BeforeEach(func() { + configSpec = nil + }) + + It("creates VM", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + }) + + Context("ConfigSpec specifies hardware spec", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{ + Name: "config-spec-name-is-not-used", + NumCPUs: 7, + MemoryMB: 5102, + } + }) + + It("CPU and memory from ConfigSpec are ignored", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Summary.Config.Name).To(Equal(vm.Name)) + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.NumCpu).ToNot(BeEquivalentTo(configSpec.NumCPUs)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + Expect(o.Summary.Config.MemorySizeMB).ToNot(BeEquivalentTo(configSpec.MemoryMB)) + }) + }) + + Context("VM Class spec CPU reservation & limits are non-zero and ConfigSpec specifies CPU reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Cpu = resource.MustParse("2") + vmClass.Spec.Policies.Resources.Limits.Cpu = resource.MustParse("3") + + // Specify a CPU reservation via ConfigSpec. This value should not be honored. + configSpec = &types.VirtualMachineConfigSpec{ + CpuAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(6), + }, + } + }) + + It("VM gets CPU reservation from VM Class spec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + resources := &vmClass.Spec.Policies.Resources + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.CPUQuantityToMhz(resources.Requests.Cpu, vcsimCPUFreq))) + Expect(*reservation).ToNot(Equal(*configSpec.CpuAllocation.Reservation)) + + limit := o.Config.CpuAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.CPUQuantityToMhz(resources.Limits.Cpu, vcsimCPUFreq))) + }) + }) + + Context("VM Class spec CPU reservation is zero and ConfigSpec specifies CPU reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Cpu = resource.MustParse("0") + vmClass.Spec.Policies.Resources.Limits.Cpu = resource.MustParse("0") + + // Specify a CPU reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + CpuAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(6), + }, + } + }) + + It("VM gets CPU reservation from ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).ToNot(BeZero()) + Expect(*reservation).To(Equal(*configSpec.CpuAllocation.Reservation)) + }) + }) + + Context("VM Class spec Memory reservation & limits are non-zero and ConfigSpec specifies memory reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Memory = resource.MustParse("4Mi") + vmClass.Spec.Policies.Resources.Limits.Memory = resource.MustParse("4Mi") + + // Specify a Memory reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + MemoryAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(5120), + }, + } + }) + + It("VM gets memory reservation from VM Class spec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + resources := &vmClass.Spec.Policies.Resources + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.MemoryQuantityToMb(resources.Requests.Memory))) + Expect(*reservation).ToNot(Equal(*configSpec.MemoryAllocation.Reservation)) + + limit := o.Config.MemoryAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.MemoryQuantityToMb(resources.Limits.Memory))) + }) + }) + + Context("VM Class spec Memory reservations are zero and ConfigSpec specifies memory reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Memory = resource.MustParse("0Mi") + vmClass.Spec.Policies.Resources.Limits.Memory = resource.MustParse("0Mi") + + // Specify a Memory reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + MemoryAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(5120), + }, + } + }) + + It("VM gets memory reservation from ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).ToNot(BeZero()) + Expect(*reservation).To(Equal(*configSpec.MemoryAllocation.Reservation)) + }) + }) + + Context("VM Class ConfigSpec specifies a network interface", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + // Create the ConfigSpec with an ethernet card. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualE1000{ + VirtualEthernetCard: ethCard, + }, + }, + }, + } + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with the NIC specified in ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + + ethDevice, ok := l[0].(*types.VirtualE1000) + // ethDevice, ok := l[0+1].(*types.VirtualE1000) + Expect(ok).To(BeTrue()) + Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) + Expect(ethDevice.MacAddress).To(Equal(ethCard.MacAddress)) + + Expect(dev.DeviceInfo).To(Equal(ethCard.VirtualDevice.DeviceInfo)) + Expect(dev.DeviceGroupInfo).To(Equal(ethCard.VirtualDevice.DeviceGroupInfo)) + Expect(dev.SlotInfo).To(Equal(ethCard.VirtualDevice.SlotInfo)) + Expect(dev.ControllerKey).To(Equal(ethCard.VirtualDevice.ControllerKey)) + Expect(ethDevice.ResourceAllocation).ToNot(BeNil()) + Expect(ethDevice.ResourceAllocation.Reservation).ToNot(BeNil()) + Expect(*ethDevice.ResourceAllocation.Reservation).To(Equal(*ethCard.ResourceAllocation.Reservation)) + }) + }) + + Context("ConfigSpec does not specify any network interfaces", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + configSpec = &types.VirtualMachineConfigSpec{} + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with the default NIC settings from provider", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + }) + }) + + Context("VM Class Spec and ConfigSpec both contain GPU and DirectPath devices", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.Devices = vmopv1.VirtualDevices{ + VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "profile-from-class", + }, + }, + DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 50, + DeviceID: 51, + CustomLabel: "label-from-class", + }, + }, + } + + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-config-spec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-config-spec", + }, + }, + }, + }, + }, + } + }) + + It("GPU and DirectPath devices from VM Class Spec.Devices are ignored", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("profile-from-config-spec")) + + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(52))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(53))) + Expect(pciBacking2.CustomLabel).To(Equal("label-from-config-spec")) + }) + }) + + Context("VM Class Config specifies an ethCard, a GPU and a DDPIO device", func() { + + BeforeEach(func() { + // Create the ConfigSpec with an ethernet card, a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualE1000{ + VirtualEthernetCard: ethCard, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "SampleLabel2", + }, + }, + }, + }, + }, + } + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with a NIC, GPU and DDPIO device specified in ConfigSpec", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + + ethDevice, ok := l[0].(*types.VirtualE1000) + // ethDevice, ok := l[0+1].(*types.VirtualE1000) + Expect(ok).To(BeTrue()) + Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) + Expect(dev.DeviceInfo).To(Equal(ethCard.VirtualDevice.DeviceInfo)) + Expect(dev.DeviceGroupInfo).To(Equal(ethCard.VirtualDevice.DeviceGroupInfo)) + Expect(dev.SlotInfo).To(Equal(ethCard.VirtualDevice.SlotInfo)) + Expect(dev.ControllerKey).To(Equal(ethCard.VirtualDevice.ControllerKey)) + Expect(ethDevice.MacAddress).To(Equal(ethCard.MacAddress)) + Expect(ethDevice.ResourceAllocation).ToNot(BeNil()) + Expect(ethDevice.ResourceAllocation.Reservation).ToNot(BeNil()) + Expect(*ethDevice.ResourceAllocation.Reservation).To(Equal(*ethCard.ResourceAllocation.Reservation)) + + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("SampleProfile2")) + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(52))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(53))) + Expect(pciBacking2.CustomLabel).To(Equal("SampleLabel2")) + + // CPU and memory should be from vm class + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + }) + + Context("VM Class Config specifies disks, disk controllers, other miscellaneous devices", func() { + BeforeEach(func() { + // Create the ConfigSpec with disks, disk controller and some misc devices: pointing device, + // video card, etc. This works fine with vcsim and helps with testing adding misc devices. + // The simulator can still reconfigure the VM with default device types like pointing devices, + // keyboard, video card, etc. But VC has some restrictions with reconfiguring a VM with new + // default device types via ConfigSpec and are usually ignored. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPointingDevice{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPointingDeviceDeviceBackingInfo{ + HostPointingDevice: "autodetect", + }, + Key: 700, + ControllerKey: 300, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPS2Controller{ + VirtualController: types.VirtualController{ + Device: []int32{700}, + VirtualDevice: types.VirtualDevice{ + Key: 300, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualMachineVideoCard{ + UseAutoDetect: pointer.Bool(false), + NumDisplays: 1, + VirtualDevice: types.VirtualDevice{ + Key: 500, + ControllerKey: 100, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIController{ + VirtualController: types.VirtualController{ + Device: []int32{500}, + VirtualDevice: types.VirtualDevice{ + Key: 100, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSCSIController{ + VirtualController: types.VirtualController{ + Device: []int32{-42}, + }, + }, + }, + }, + } + }) + + // FIXME: vcsim behavior needs to be closer to real VC here so there aren't dupes + It("Reconfigures the VM with all misc devices in ConfigSpec except disk and disk controllers", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + + // VM already has a default pointing device and the spec adds one more + // info about the default device is unknown to assert on + pointingDev := devList.SelectByType(&types.VirtualPointingDevice{}) + Expect(pointingDev).To(HaveLen(2)) + dev := pointingDev[0].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualPointingDeviceDeviceBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backing.HostPointingDevice).To(Equal("autodetect")) + Expect(dev.Key).To(Equal(int32(700))) + Expect(dev.ControllerKey).To(Equal(int32(300))) + + ps2Controllers := devList.SelectByType(&types.VirtualPS2Controller{}) + Expect(ps2Controllers).To(HaveLen(1)) + dev = ps2Controllers[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(300))) + + pciControllers := devList.SelectByType(&types.VirtualPCIController{}) + Expect(pciControllers).To(HaveLen(1)) + dev = pciControllers[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(100))) + + // VM already has a default video card and the spec adds one more + // info about the default device is unknown to assert on + video := devList.SelectByType(&types.VirtualMachineVideoCard{}) + Expect(video).To(HaveLen(2)) + dev = video[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(500))) + Expect(dev.ControllerKey).To(Equal(int32(100))) + + // Disk and disk controllers from config spec should not get added, since we + // filter them out in our ConfigSpec + diskControllers := devList.SelectByType(&types.VirtualSCSIController{}) + Expect(diskControllers).To(BeEmpty()) + + // Only preexisting disk should be present on VM -- len: 1 + disks := devList.SelectByType(&types.VirtualDisk{}) + Expect(disks).To(HaveLen(1)) + dev = disks[0].GetVirtualDevice() + Expect(dev.Key).ToNot(Equal(int32(-42))) + }) + }) + + Context("VM Class Config does not specify a hardware version", func() { + + Context("VM Class has vGPU and/or DDPIO devices", func() { + BeforeEach(func() { + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PCI devices", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPCIPassthruDevices))) + }) + }) + + Context("VM Class has vGPU and/or DDPIO devices and VM spec has a PVC", func() { + BeforeEach(func() { + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + } + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "dummy-vol", + Attached: true, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PCI devices", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPCIPassthruDevices))) + }) + }) + + Context("VM spec has a PVC", func() { + BeforeEach(func() { + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + } + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "dummy-vol", + Attached: true, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PVCs", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPVC))) + }) + }) + }) + + Context("VMClassAsConfig FSS is Enabled", func() { + + BeforeEach(func() { + testConfig.WithVMClassAsConfig = true + }) + + When("configSpec has disk and disk controllers", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSATAController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 101, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSCSIController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 103, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualNVMEController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 104, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + }, + }, + } + }) + + It("creates a VM with disk controllers", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + satacont := devList.SelectByType(&types.VirtualSATAController{}) + Expect(satacont).To(HaveLen(1)) + dev := satacont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(101))) + + scsicont := devList.SelectByType(&types.VirtualSCSIController{}) + Expect(scsicont).To(HaveLen(1)) + dev = scsicont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(103))) + + nvmecont := devList.SelectByType(&types.VirtualNVMEController{}) + Expect(nvmecont).To(HaveLen(1)) + dev = nvmecont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(104))) + + // only preexisting disk should be present on VM -- len: 1 + disks := devList.SelectByType(&types.VirtualDisk{}) + Expect(disks).To(HaveLen(1)) + dev1 := disks[0].GetVirtualDevice() + Expect(dev1.Key).ToNot(Equal(int32(-42))) + }) + }) + }) + }) + + Context("CreateOrUpdate VM", func() { + + It("Basic VM", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected Status values", func() { + Expect(vm.Status.PowerState).To(Equal(vm.Spec.PowerState)) + Expect(vm.Status.Host).ToNot(BeEmpty()) + Expect(vm.Status.InstanceUUID).To(And(Not(BeEmpty()), Equal(o.Config.InstanceUuid))) + Expect(vm.Status.BiosUUID).To(And(Not(BeEmpty()), Equal(o.Config.Uuid))) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionClassReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionStorageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix(fmt.Sprintf("/%s/%s", nsInfo.Namespace, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + + By("has expected power state", func() { + Expect(o.Summary.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + }) + + vmClassRes := &vmClass.Spec.Policies.Resources + + By("has expected CpuAllocation", func() { + Expect(o.Config.CpuAllocation).ToNot(BeNil()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.CPUQuantityToMhz(vmClassRes.Requests.Cpu, vcsimCPUFreq))) + limit := o.Config.CpuAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.CPUQuantityToMhz(vmClassRes.Limits.Cpu, vcsimCPUFreq))) + }) + + By("has expected MemoryAllocation", func() { + Expect(o.Config.MemoryAllocation).ToNot(BeNil()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.MemoryQuantityToMb(vmClassRes.Requests.Memory))) + limit := o.Config.MemoryAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.MemoryQuantityToMb(vmClassRes.Limits.Memory))) + }) + + By("has expected hardware config", func() { + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + + // TODO: More assertions! + }) + + Context("VM Class with PCI passthrough devices", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.Devices = vmopv1.VirtualDevices{ + VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "profile-from-class-without-class-as-config-fss", + }, + }, + DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 59, + DeviceID: 60, + CustomLabel: "label-from-class-without-class-as-config-fss", + }, + }, + } + }) + + It("VM should have expected PCI devices from VM Class", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("profile-from-class-without-class-as-config-fss")) + + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(59))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(60))) + Expect(pciBacking2.CustomLabel).To(Equal("label-from-class-without-class-as-config-fss")) + }) + }) + + Context("Without Storage Class", func() { + BeforeEach(func() { + testConfig.WithoutStorageClass = true + }) + + It("Creates VM", func() { + Expect(vm.Spec.StorageClass).To(BeEmpty()) + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected datastore", func() { + datastore, err := ctx.Finder.DefaultDatastore(ctx) + Expect(err).ToNot(HaveOccurred()) + + Expect(o.Datastore).To(HaveLen(1)) + Expect(o.Datastore[0]).To(Equal(datastore.Reference())) + }) + }) + }) + + Context("Without Content Library", func() { + BeforeEach(func() { + testConfig.WithContentLibrary = false + }) + + // TODO: Dedupe this with "Basic VM" above + It("Clones VM", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected Status values", func() { + Expect(vm.Status.PowerState).To(Equal(vm.Spec.PowerState)) + Expect(vm.Status.Host).ToNot(BeEmpty()) + Expect(vm.Status.InstanceUUID).To(And(Not(BeEmpty()), Equal(o.Config.InstanceUuid))) + Expect(vm.Status.BiosUUID).To(And(Not(BeEmpty()), Equal(o.Config.Uuid))) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionClassReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionStorageReady)).To(BeTrue()) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix(fmt.Sprintf("/%s/%s", nsInfo.Namespace, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + + By("has expected power state", func() { + Expect(o.Summary.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + }) + + By("has expected hardware config", func() { + // TODO: Fix vcsim behavior: NumCPU is correct "2" in the CloneSpec.Config but ends up + // with 1 CPU from source VM. Ditto for MemorySize. These assertions are only working + // because the state is on so we reconfigure the VM after it is created. + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + + // TODO: More assertions! + }) + }) + + // BMV: I don't think this is actually supported. + XIt("Create VM from VMTX in ContentLibrary", func() { + imageName := "test-vm-vmtx" + + ctx.ContentLibraryItemTemplate("DC0_C0_RP0_VM0", imageName) + vm.Spec.ImageName = imageName + + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("When fault domains is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + It("creates VM in placement selected zone", func() { + Expect(vm.Labels).ToNot(HaveKey(topology.KubernetesTopologyZoneLabelKey)) + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + azName, ok := vm.Labels[topology.KubernetesTopologyZoneLabelKey] + Expect(ok).To(BeTrue()) + Expect(azName).To(BeElementOf(ctx.ZoneNames)) + + By("VM is created in the zone's ResourcePool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, azName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + }) + + It("creates VM in assigned zone", func() { + azName := ctx.ZoneNames[rand.Intn(len(ctx.ZoneNames))] //nolint:gosec + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = azName + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + By("VM is created in the zone's ResourcePool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, azName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + }) + }) + + Context("When Instance Storage FSS is enabled", func() { + BeforeEach(func() { + testConfig.WithInstanceStorage = true + }) + + expectInstanceStorageVolumes := func( + vm *vmopv1.VirtualMachine, + isStorage vmopv1.InstanceStorage) { + + ExpectWithOffset(1, isStorage.Volumes).ToNot(BeEmpty()) + isVolumes := instancestorage.FilterVolumes(vm) + ExpectWithOffset(1, isVolumes).To(HaveLen(len(isStorage.Volumes))) + + for _, isVol := range isStorage.Volumes { + found := false + + for idx, vol := range isVolumes { + claim := vol.PersistentVolumeClaim.InstanceVolumeClaim + if claim.StorageClass == isStorage.StorageClass && claim.Size == isVol.Size { + isVolumes = append(isVolumes[:idx], isVolumes[idx+1:]...) + found = true + break + } + } + + ExpectWithOffset(1, found).To(BeTrue(), "failed to find instance storage volume for %v", isVol) + } + } + + It("creates VM without instance storage", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + It("create VM with instance storage", func() { + Expect(vm.Spec.Volumes).To(BeEmpty()) + + vmClass.Spec.Hardware.InstanceStorage = vmopv1.InstanceStorage{ + StorageClass: vm.Spec.StorageClass, + Volumes: []vmopv1.InstanceStorageVolume{ + { + Size: resource.MustParse("256Gi"), + }, + { + Size: resource.MustParse("512Gi"), + }, + }, + } + Expect(ctx.Client.Update(ctx, vmClass)).To(Succeed()) + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("instance storage PVCs are not bound yet")) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeFalse()) + + By("Instance storage volumes should be added to VM", func() { + Expect(instancestorage.IsPresent(vm)).To(BeTrue()) + expectInstanceStorageVolumes(vm, vmClass.Spec.Hardware.InstanceStorage) + }) + + By("Placement should have been done", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionPlacementReady)).To(BeTrue()) + Expect(vm.Annotations).To(HaveKey(constants.InstanceStorageSelectedNodeAnnotationKey)) + Expect(vm.Annotations).To(HaveKey(constants.InstanceStorageSelectedNodeMOIDAnnotationKey)) + }) + + isVol0 := vm.Spec.Volumes[0] + Expect(isVol0.PersistentVolumeClaim.InstanceVolumeClaim).ToNot(BeNil()) + + By("simulate volume controller workflow", func() { + // Simulate what would be set by volume controller. + vm.Annotations[constants.InstanceStoragePVCsBoundAnnotationKey] = "" + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("status update pending for persistent volume: %s on VM", isVol0.Name))) + + // Simulate what would be set by the volume controller. + for _, vol := range vm.Spec.Volumes { + vm.Status.Volumes = append(vm.Status.Volumes, vmopv1.VirtualMachineVolumeStatus{ + Name: vol.Name, + Attached: true, + }) + } + }) + + By("VM is now created", func() { + _, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + }) + }) + + It("Powers VM off", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOff)) + }) + + It("returns error when StorageClass is required but none specified", func() { + vm.Spec.StorageClass = "" + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("StorageClass is required but not specified")) + }) + + It("Can be called multiple times", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + modified := o.Config.Modified + + _, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + // Try to assert nothing changed. + Expect(o.Config.Modified).To(Equal(modified)) + }) + + Context("VM Metadata", func() { + + Context("ExtraConfig Transport", func() { + var ec map[string]interface{} + + JustBeforeEach(func() { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "md-configmap-", + Namespace: vm.Namespace, + }, + Data: map[string]string{ + "foo.bar": "should-be-ignored", + "guestinfo.Foo": "foo", + }, + } + Expect(ctx.Client.Create(ctx, configMap)).To(Succeed()) + + /* + vm.Spec.VmMetadata = &vmopv1.VirtualMachineMetadata{ + ConfigMapName: configMap.Name, + Transport: vmopv1.VirtualMachineMetadataExtraConfigTransport, + } + */ + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + ec = map[string]interface{}{} + for _, option := range o.Config.ExtraConfig { + if val := option.GetOptionValue(); val != nil { + ec[val.Key] = val.Value.(string) + } + } + }) + + AfterEach(func() { + ec = nil + }) + + // TODO: As is we can't really honor "guestinfo.*" prefix + XIt("Metadata data is included in ExtraConfig", func() { + Expect(ec).ToNot(HaveKey("foo.bar")) + Expect(ec).To(HaveKeyWithValue("guestinfo.Foo", "foo")) + + By("Should include default keys and values", func() { + Expect(ec).To(HaveKeyWithValue("disk.enableUUID", "TRUE")) + Expect(ec).To(HaveKeyWithValue("vmware.tools.gosc.ignoretoolscheck", "TRUE")) + }) + }) + + Context("JSON_EXTRA_CONFIG is specified", func() { + BeforeEach(func() { + b, err := json.Marshal( + struct { + Foo string + Bar string + }{ + Foo: "f00", + Bar: "42", + }, + ) + Expect(err).ToNot(HaveOccurred()) + testConfig.WithJSONExtraConfig = string(b) + }) + + It("Global config is included in ExtraConfig", func() { + Expect(ec).To(HaveKeyWithValue("Foo", "f00")) + Expect(ec).To(HaveKeyWithValue("Bar", "42")) + }) + }) + }) + }) + + Context("Network", func() { + + It("Should not have a nic", func() { + Expect(vm.Spec.Network.Disabled).To(BeTrue()) + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(BeEmpty()) + }) + + Context("Multiple NICs are specified", func() { + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + vm.Spec.Network.Disabled = false + vm.Spec.Network.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: "VM Network"}, + }, + { + Name: "eth1", + Network: common.PartialObjectRef{Name: dvpgName}, + }, + } + }) + + It("Has expected devices", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(2)) + + dev1 := l[0].GetVirtualDevice() + backing1, ok := dev1.Backing.(*types.VirtualEthernetCardNetworkBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backing1.DeviceName).To(Equal("VM Network")) + + dev2 := l[1].GetVirtualDevice() + backing2, ok := dev2.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing2.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + }) + }) + }) + + Context("Disks", func() { + + Context("VM has thin provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThin + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.ThinProvisioned).To(PointTo(BeTrue())) + }) + }) + + XContext("VM has thick provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThick + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + /* vcsim CL deploy has "thick" but that isn't reflected for this disk. */ + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.ThinProvisioned).To(PointTo(BeFalse())) + }) + }) + + XContext("VM has eager zero provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThickEagerZero + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + /* vcsim CL deploy has "eagerZeroedThick" but that isn't reflected for this disk. */ + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.EagerlyScrub).To(PointTo(BeTrue())) + }) + }) + + Context("Should resize root disk", func() { + newSize := resource.MustParse("4242Gi") + + It("Succeeds", func() { + vm.Spec.Advanced.BootDiskCapacity = newSize + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + disk, _ := getVMHomeDisk(ctx, vcVM, o) + Expect(disk.CapacityInBytes).To(BeEquivalentTo(newSize.Value())) + }) + }) + }) + + Context("CNS Volumes", func() { + cnsVolumeName := "cns-volume-1" + + It("CSI Volumes workflow", func() { + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn + By("Add CNS volume to VM", func() { + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: cnsVolumeName, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-volume-1", + }, + }, + }, + }, + } + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("status update pending for persistent volume: %s on VM", cnsVolumeName))) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + }) + + By("CNS volume is not attached", func() { + errMsg := "blah blah blah not attached" + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: cnsVolumeName, + Attached: false, + Error: errMsg, + }, + } + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("persistent volume: %s not attached to VM", cnsVolumeName))) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + }) + + By("CNS volume is attached", func() { + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: cnsVolumeName, + Attached: true, + }, + } + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + }) + }) + }) + + Context("When fault domains is enabled", func() { + const zoneName = "az-1" + + BeforeEach(func() { + testConfig.WithFaultDomains = true + // Explicitly place the VM into one of the zones that the test context will create. + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("Reverse lookups existing VM into correct zone", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + Expect(vm.Status.Zone).To(Equal(zoneName)) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + Expect(vm.Status.Zone).To(Equal(zoneName)) + }) + }) + }) + + Context("VM SetResourcePolicy", func() { + var resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + + JustBeforeEach(func() { + resourcePolicyName := "test-policy" + resourcePolicy = getVirtualMachineSetResourcePolicy(resourcePolicyName, nsInfo.Namespace) + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(ctx.Client.Create(ctx, resourcePolicy)).To(Succeed()) + + vm.Annotations["vsphere-cluster-module-group"] = resourcePolicy.Spec.ClusterModuleGroups[0] + vm.Spec.Reserved.ResourcePolicyName = resourcePolicy.Name + }) + + AfterEach(func() { + resourcePolicy = nil + }) + + It("VM is created in child Folder and ResourcePool", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix( + fmt.Sprintf("/%s/%s/%s", nsInfo.Namespace, resourcePolicy.Spec.Folder, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + childRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", resourcePolicy.Spec.ResourcePool.Name) + Expect(childRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(childRP.Reference().Value)) + }) + }) + + It("Cluster Modules", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + members, err := cluster.NewManager(ctx.RestClient).ListModuleMembers(ctx, resourcePolicy.Status.ClusterModules[0].ModuleUuid) + Expect(err).ToNot(HaveOccurred()) + Expect(members).To(ContainElements(vcVM.Reference())) + }) + + It("Returns error with non-existence cluster module", func() { + vm.Annotations["vsphere-cluster-module-group"] = "bogusClusterMod" + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("ClusterModule bogusClusterMod not found")) + }) + }) + + Context("Delete VM", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + Context("when the VM is off", func() { + BeforeEach(func() { + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + }) + + It("deletes the VM", func() { + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + }) + + It("when the VM is on", func() { + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + // This checks that we power off the VM prior to deletion. + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + + It("returns success when VM does not exist", func() { + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + }) + + Context("When fault domains is enabled", func() { + const zoneName = "az-1" + + BeforeEach(func() { + testConfig.WithFaultDomains = true + // Explicitly place the VM into one of the zones that the test context will create. + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("returns NotFound when VM does not exist", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("Deletes existing VM when zone info is missing", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + }) + }) + + Context("Guest Heartbeat", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return guest heartbeat", func() { + heartbeat, err := vmProvider.GetVirtualMachineGuestHeartbeat(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + // Just testing for property query: field not set in vcsim. + Expect(heartbeat).To(BeEmpty()) + }) + }) + + Context("Web console ticket", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return ticket", func() { + // vcsim doesn't implement this yet so expect an error. + _, err := vmProvider.GetVirtualMachineWebMKSTicket(ctx, vm, "foo") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not implement: AcquireTicket")) + }) + }) + + Context("VM hardware version", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return version", func() { + version, err := vmProvider.GetVirtualMachineHardwareVersion(ctx, vm) + Expect(err).NotTo(HaveOccurred()) + Expect(version).To(Equal(int32(9))) + }) + }) + }) +} + +// getVMHomeDisk gets the VM's "home" disk. It makes some assumptions about the backing and disk name. +func getVMHomeDisk( + ctx *builder.TestContextForVCSim, + vcVM *object.VirtualMachine, + o mo.VirtualMachine) (*types.VirtualDisk, *types.VirtualDiskFlatVer2BackingInfo) { + + ExpectWithOffset(1, vcVM.Name()).ToNot(BeEmpty()) + ExpectWithOffset(1, o.Datastore).ToNot(BeEmpty()) + var dso mo.Datastore + ExpectWithOffset(1, vcVM.Properties(ctx, o.Datastore[0], nil, &dso)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByBackingInfo(&types.VirtualDiskFlatVer2BackingInfo{ + VirtualDeviceFileBackingInfo: types.VirtualDeviceFileBackingInfo{ + FileName: fmt.Sprintf("[%s] %s/disk-0.vmdk", dso.Name, vcVM.Name()), + }, + }) + ExpectWithOffset(1, l).To(HaveLen(1)) + + disk := l[0].(*types.VirtualDisk) + backing := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo) + + return disk, backing +} + +//nolint:unparam +func getDVPG( + ctx *builder.TestContextForVCSim, + path string) (object.NetworkReference, *object.DistributedVirtualPortgroup) { + + network, err := ctx.Finder.Network(ctx, path) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + dvpg, ok := network.(*object.DistributedVirtualPortgroup) + ExpectWithOffset(1, ok).To(BeTrue()) + + return network, dvpg +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go new file mode 100644 index 000000000..5506214e8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go @@ -0,0 +1,335 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + "encoding/json" + "fmt" + + "github.com/google/uuid" + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" +) + +// TODO: This mostly just a placeholder until we spend time on something better. Individual types +// don't make much sense since we don't lump everything under a single prereq condition anymore. +func errToConditionReasonAndMessage(err error) (string, string) { + switch { + case apierrors.IsNotFound(err): + return "NotFound", err.Error() + case apierrors.IsForbidden(err): + return "Forbidden", err.Error() + case apierrors.IsInvalid(err): + return "Invalid", err.Error() + case apierrors.IsInternalError(err): + return "InternalError", err.Error() + default: + return "GetError", err.Error() + } +} + +func GetVirtualMachineClass( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (*vmopv1.VirtualMachineClass, error) { + + key := ctrlclient.ObjectKey{Name: vmCtx.VM.Spec.ClassName, Namespace: vmCtx.VM.Namespace} + vmClass := &vmopv1.VirtualMachineClass{} + if err := k8sClient.Get(vmCtx, key, vmClass); err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady, reason, msg) + return nil, err + } + + if !vmClass.Status.Ready { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady, + "NotReady", "VirtualMachineClass is not marked as Ready") + return nil, fmt.Errorf("VirtualMachineClass is not Ready") + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady) + + return vmClass, nil +} + +func GetVirtualMachineImageSpecAndStatus( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (ctrlclient.Object, *vmopv1.VirtualMachineImageSpec, *vmopv1.VirtualMachineImageStatus, error) { + + var obj ctrlclient.Object + var spec *vmopv1.VirtualMachineImageSpec + var status *vmopv1.VirtualMachineImageStatus + + key := ctrlclient.ObjectKey{Name: vmCtx.VM.Spec.ImageName, Namespace: vmCtx.VM.Namespace} + vmImage := &vmopv1.VirtualMachineImage{} + if err := k8sClient.Get(vmCtx, key, vmImage); err != nil { + clusterVMImage := &vmopv1.ClusterVirtualMachineImage{} + + if apierrors.IsNotFound(err) { + key.Namespace = "" + err = k8sClient.Get(vmCtx, key, clusterVMImage) + } + + if err != nil { + // Don't use the k8s error as-is as we don't know to prefer the NS or cluster scoped error message. + // This is the same error/message that the prior code used. + reason, msg := "NotFound", fmt.Sprintf("Failed to get the VM's image: %s", key.Name) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, reason, msg) + return nil, nil, nil, fmt.Errorf("%s: %w", msg, err) + } + + obj, spec, status = clusterVMImage, &clusterVMImage.Spec, &clusterVMImage.Status + } else { + obj, spec, status = vmImage, &vmImage.Spec, &vmImage.Status + } + + // TODO: Fix the image conditions so it just has a single Ready instead of bleeding the CL stuff. + if !conditions.IsTrueFromConditions(status.Conditions, vmopv1.VirtualMachineImageSyncedCondition) { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, + "NotReady", "VirtualMachineImage is not ready") + return nil, nil, nil, fmt.Errorf("VirtualMachineImage is not ready") + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady) + + return obj, spec, status, nil +} + +func getSecretData( + vmCtx context.VirtualMachineContextA2, + name string, + cmFallback bool, + k8sClient ctrlclient.Client) (map[string]string, error) { + + var data map[string]string + + key := ctrlclient.ObjectKey{Name: name, Namespace: vmCtx.VM.Namespace} + secret := &corev1.Secret{} + if err := k8sClient.Get(vmCtx, key, secret); err != nil { + configMap := &corev1.ConfigMap{} + + // For backwards compat if we cannot find the Secret, fallback to a ConfigMap. In v1a1, either a + // Secret and ConfigMap was supported for metadata (bootstrap) as separate fields, but v1a2 only + // supports Secrets. + if cmFallback && apierrors.IsNotFound(err) { + err = k8sClient.Get(vmCtx, key, configMap) + } + + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, err + } + + data = configMap.Data + } else { + data = make(map[string]string, len(secret.Data)) + + for k, v := range secret.Data { + data[k] = string(v) + } + } + + return data, nil +} + +func GetVirtualMachineBootstrap( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (map[string]string, map[string]string, map[string]map[string]string, error) { + + bootstrapSpec := &vmCtx.VM.Spec.Bootstrap + var secretName string + var data, vAppData map[string]string + var vAppExData map[string]map[string]string + + if cloudInit := bootstrapSpec.CloudInit; cloudInit != nil { + secretName = cloudInit.RawCloudConfig.Name + } else if sysPrep := bootstrapSpec.Sysprep; sysPrep != nil { + secretName = sysPrep.RawSysprep.Name + } + + if secretName != "" { + var err error + + data, err = getSecretData(vmCtx, secretName, true, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + } + + // vApp bootstrap can be used alongside LinuxPrep/Sysprep. + if vApp := bootstrapSpec.VAppConfig; vApp != nil { + + if vApp.RawProperties != "" { + var err error + + vAppData, err = getSecretData(vmCtx, vApp.RawProperties, true, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + + } else { + for _, p := range vApp.Properties { + from := p.Value.From + if from == nil { + continue + } + + if _, ok := vAppExData[from.Name]; !ok { + // Do the easy thing here and carry along each Secret's entire data. We could instead + // shoehorn this in the vAppData with a concat key using an invalid k8s name delimiter. + // TODO: Check that key exists, and/or deal with from.Optional. Too many options. + fromData, err := getSecretData(vmCtx, from.Name, false, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + + if vAppExData == nil { + vAppExData = make(map[string]map[string]string) + } + vAppExData[from.Name] = fromData + } + } + } + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady) + + return data, vAppData, vAppExData, nil +} + +func GetVMSetResourcePolicy( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (*vmopv1.VirtualMachineSetResourcePolicy, error) { + + rpName := vmCtx.VM.Spec.Reserved.ResourcePolicyName + if rpName == "" { + conditions.Delete(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady) + return nil, nil + } + + key := ctrlclient.ObjectKey{Name: rpName, Namespace: vmCtx.VM.Namespace} + resourcePolicy := &vmopv1.VirtualMachineSetResourcePolicy{} + if err := k8sClient.Get(vmCtx, key, resourcePolicy); err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady, reason, msg) + return nil, err + } + + // The VirtualMachineSetResourcePolicy doesn't have a Ready condition or field but don't + // allow a VM to use a policy that's being deleted. + if !resourcePolicy.DeletionTimestamp.IsZero() { + err := fmt.Errorf("VirtualMachineSetResourcePolicy is being deleted") + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady, + "NotReady", err.Error()) + return nil, err + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady) + + return resourcePolicy, nil +} + +// AddInstanceStorageVolumes checks if VM class is configured with instance storage volumes and appends the +// volumes to the VM's Spec if not already done. Return true if the VM had or now has instance storage volumes. +func AddInstanceStorageVolumes( + vmCtx context.VirtualMachineContextA2, + vmClass *vmopv1.VirtualMachineClass) bool { + + if instancestorage.IsPresent(vmCtx.VM) { + // Instance storage disks are copied from the class to the VM only once, regardless + // if the class changes. + return true + } + + is := vmClass.Spec.Hardware.InstanceStorage + if len(is.Volumes) == 0 { + return false + } + + volumes := make([]vmopv1.VirtualMachineVolume, 0, len(is.Volumes)) + + for _, isv := range is.Volumes { + name := constants.InstanceStoragePVCNamePrefix + uuid.NewString() + + vmv := vmopv1.VirtualMachineVolume{ + Name: name, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + ReadOnly: false, + }, + InstanceVolumeClaim: &vmopv1.InstanceVolumeClaimVolumeSource{ + StorageClass: is.StorageClass, + Size: isv.Size, + }, + }, + }, + } + volumes = append(volumes, vmv) + } + + vmCtx.VM.Spec.Volumes = append(vmCtx.VM.Spec.Volumes, volumes...) + return true +} + +func GetVMClassConfigSpec(raw json.RawMessage) (*types.VirtualMachineConfigSpec, error) { + classConfigSpec, err := util.UnmarshalConfigSpecFromJSON(raw) + if err != nil { + return nil, err + } + util.SanitizeVMClassConfigSpec(classConfigSpec) + + return classConfigSpec, nil +} + +// HasPVC returns true if the VirtualMachine spec has a Persistent Volume claim. +func HasPVC(vmSpec vmopv1.VirtualMachineSpec) bool { + for _, vol := range vmSpec.Volumes { + if vol.PersistentVolumeClaim != nil { + return true + } + } + return false +} + +// HardwareVersionForPVCandPCIDevices returns a hardware version for VMs with PVCs and PCI devices(vGPUs/DDPIO devices) +// The hardware version is determined based on the below criteria: VMs with +// - Persistent Volume Claim (PVC) get the max(the image hardware version, minimum supported virtual hardware version for persistent volumes) +// - vGPUs/DDPIO devices get the max(the image hardware version, minimum supported virtual hardware version for PCI devices) +// - Both vGPU/DDPIO devices and PVCs get the max(the image hardware version, minimum supported virtual hardware version for PCI devices) +// - none of the above returns 0. +func HardwareVersionForPVCandPCIDevices(imageHWVersion int32, configSpec *types.VirtualMachineConfigSpec, hasPVC bool) int32 { + var configSpecHWVersion int32 + configSpecDevs := util.DevicesFromConfigSpec(configSpec) + + if len(util.SelectNvidiaVgpu(configSpecDevs)) > 0 || len(util.SelectDynamicDirectPathIO(configSpecDevs)) > 0 { + configSpecHWVersion = constants.MinSupportedHWVersionForPCIPassthruDevices + if imageHWVersion != 0 && imageHWVersion > constants.MinSupportedHWVersionForPCIPassthruDevices { + configSpecHWVersion = imageHWVersion + } + } else if hasPVC { + configSpecHWVersion = constants.MinSupportedHWVersionForPVC + if imageHWVersion != 0 && imageHWVersion > constants.MinSupportedHWVersionForPVC { + configSpecHWVersion = imageHWVersion + } + } + + return configSpecHWVersion +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go new file mode 100644 index 000000000..cf7fe642d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go @@ -0,0 +1,629 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + goctx "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vmUtilTests() { + + var ( + k8sClient client.Client + initObjects []client.Object + + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + vm := builder.DummyBasicVirtualMachineA2("test-vm", "dummy-ns") + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.WithValue(goctx.Background(), context.MaxDeployThreadsContextKey, 16), + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + JustBeforeEach(func() { + k8sClient = builder.NewFakeClient(initObjects...) + }) + + AfterEach(func() { + k8sClient = nil + initObjects = nil + }) + + Context("GetVirtualMachineClass", func() { + oldNamespacedVMClassFSSEnabledFunc := lib.IsNamespacedVMClassFSSEnabled + + // NOTE: As we currently have it, v1a2 must have this enabled. + When("WCP_Namespaced_VM_Class FSS is enabled", func() { + var ( + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + vmClass = builder.DummyVirtualMachineClass2A2("dummy-vm-class") + vmClass.Namespace = vmCtx.VM.Namespace + vmCtx.VM.Spec.ClassName = vmClass.Name + + lib.IsNamespacedVMClassFSSEnabled = func() bool { + return true + } + }) + + AfterEach(func() { + lib.IsNamespacedVMClassFSSEnabled = oldNamespacedVMClassFSSEnabledFunc + }) + + Context("VirtualMachineClass custom resource doesn't exist", func() { + It("Returns error and sets condition when VM Class does not exist", func() { + expectedErrMsg := fmt.Sprintf("virtualmachineclasses.vmoperator.vmware.com %q not found", vmCtx.VM.Spec.ClassName) + + _, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineConditionClassReady, "NotFound", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + Context("VirtualMachineClass custom resource exists", func() { + + When("Is not Ready", func() { + + BeforeEach(func() { + vmClass.Status.Ready = false + initObjects = append(initObjects, vmClass) + }) + + It("returns an error", func() { + _, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("VirtualMachineClass is not Ready")) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition( + vmopv1.VirtualMachineConditionClassReady, + "NotReady", + "VirtualMachineClass is not marked as Ready"), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("Is Ready", func() { + + BeforeEach(func() { + vmClass.Status.Ready = true + initObjects = append(initObjects, vmClass) + }) + + It("returns success", func() { + class, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(class).ToNot(BeNil()) + }) + }) + }) + }) + }) + + Context("GetVMImageStatusAndContentLibraryUUID", func() { + + // NOTE: As we currently have it, v1a2 must have this enabled. + When("WCPVMImageRegistry FSS is enabled", func() { + + var ( + nsVMImage *vmopv1.VirtualMachineImage + clusterVMImage *vmopv1.ClusterVirtualMachineImage + ) + + BeforeEach(func() { + nsVMImage = builder.DummyVirtualMachineImageA2("dummy-ns-vm-image") + nsVMImage.Namespace = vmCtx.VM.Namespace + conditions.MarkTrue(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + clusterVMImage = builder.DummyClusterVirtualMachineImageA2("dummy-cluster-vm-image") + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + + lib.IsWCPVMImageRegistryEnabled = func() bool { + return true + } + }) + + When("Neither cluster or namespace scoped VM image exists", func() { + + It("returns error and sets condition", func() { + _, _, _, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + expectedErrMsg := fmt.Sprintf("Failed to get the VM's image: %s", vmCtx.VM.Spec.ImageName) + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineConditionImageReady, "NotFound", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("VM image exists but the image is not ready", func() { + + BeforeEach(func() { + conditions.MarkFalse(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition, "NotReady", "") // XXX Until rollup condition + initObjects = append(initObjects, nsVMImage) + vmCtx.VM.Spec.ImageName = nsVMImage.Name + }) + + It("returns error and sets VM condition", func() { + _, _, _, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + expectedErrMsg := "VirtualMachineImage is not ready" + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition( + vmopv1.VirtualMachineConditionImageReady, "NotReady", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("Namespace scoped VirtualMachineImage exists and ready", func() { + BeforeEach(func() { + conditions.MarkTrue(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + initObjects = append(initObjects, nsVMImage) + vmCtx.VM.Spec.ImageName = nsVMImage.Name + }) + + It("returns success", func() { + imgObj, spec, status, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(imgObj).ToNot(BeNil()) + Expect(imgObj.GetObjectKind().GroupVersionKind().Kind).To(Equal("VirtualMachineImage")) + Expect(spec).ToNot(BeNil()) + Expect(status).ToNot(BeNil()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + }) + }) + + When("ClusterVirtualMachineImage exists and ready", func() { + BeforeEach(func() { + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + initObjects = append(initObjects, clusterVMImage) + vmCtx.VM.Spec.ImageName = clusterVMImage.Name + }) + + It("returns success", func() { + imgObj, spec, status, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(imgObj).ToNot(BeNil()) + Expect(imgObj.GetObjectKind().GroupVersionKind().Kind).To(Equal("ClusterVirtualMachineImage")) + Expect(spec).ToNot(BeNil()) + Expect(status).ToNot(BeNil()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + }) + }) + }) + }) + + Context("GetVirtualMachineBootstrap", func() { + const dataName = "dummy-vm-bootstrap-data" + const vAppDataName = "dummy-vm-bootstrap-vapp-data" + + var ( + bootstrapCM *corev1.ConfigMap + bootstrapSecret *corev1.Secret + bootstrapVAppCM *corev1.ConfigMap + ) + + BeforeEach(func() { + bootstrapCM = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string]string{ + "foo": "bar", + }, + } + + bootstrapSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string][]byte{ + "foo1": []byte("bar1"), + }, + } + + bootstrapVAppCM = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: vAppDataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string]string{ + "foo-vapp": "bar-vapp", + }, + } + }) + + When("Bootstrap via CloudInit", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, + } + vmCtx.VM.Spec.Bootstrap.CloudInit.RawCloudConfig.Name = dataName + }) + + It("return an error when resources does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM) + }) + + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo", "bar")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("Secret exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM, bootstrapSecret) + }) + + When("Prefers Secret over ConfigMap", func() { + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + // Prefer Secret over ConfigMap. + Expect(data).To(HaveKeyWithValue("foo1", "bar1")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + }) + }) + + When("Bootstrap via Sysprep", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + Sysprep: &vmopv1.VirtualMachineBootstrapSysprepSpec{}, + } + vmCtx.VM.Spec.Bootstrap.Sysprep.RawSysprep.Name = dataName + }) + + It("return an error when resource does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM) + }) + + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo", "bar")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("Secret exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM, bootstrapSecret) + }) + + When("Prefers Secret over ConfigMap", func() { + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo1", "bar1")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + }) + }) + + When("Bootstrap with vAppConfig", func() { + + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + VAppConfig: &vmopv1.VirtualMachineBootstrapVAppConfigSpec{}, + } + vmCtx.VM.Spec.Bootstrap.VAppConfig.RawProperties = vAppDataName + }) + + It("return an error when resource does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapVAppCM) + }) + + It("returns success", func() { + _, data, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo-vapp", "bar-vapp")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("vAppConfig with properties", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap.VAppConfig.Properties = []common.KeyValueOrSecretKeySelectorPair{ + { + Value: common.ValueOrSecretKeySelector{ + From: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: vAppDataName, + }, + Key: "foo-vapp", + }, + }, + }, + } + + It("returns success", func() { + _, _, exData, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(exData).To(HaveKey(vAppDataName)) + data := exData[vAppDataName] + Expect(data).To(HaveKeyWithValue("foo-vapp", "bar-vapp")) + }) + }) + }) + }) + }) + + Context("GetVMSetResourcePolicy", func() { + + var ( + vmResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + BeforeEach(func() { + vmResourcePolicy = &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-vm-rp", + Namespace: vmCtx.VM.Namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{Name: "fooRP"}, + Folder: "fooFolder", + }, + } + }) + + It("returns success when VM does not have SetResourcePolicy", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "" + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + + It("VM SetResourcePolicy does not exist", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "bogus" + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + + When("VM SetResourcePolicy exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, vmResourcePolicy) + vmCtx.VM.Spec.Reserved.ResourcePolicyName = vmResourcePolicy.Name + }) + + It("returns success", func() { + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).ToNot(BeNil()) + }) + }) + }) + + Context("AddInstanceStorageVolumes", func() { + + var ( + vmClass *vmopv1.VirtualMachineClass + ) + + expectInstanceStorageVolumes := func( + vm *vmopv1.VirtualMachine, + isStorage vmopv1.InstanceStorage) { + + ExpectWithOffset(1, isStorage.Volumes).ToNot(BeEmpty()) + isVolumes := instancestorage.FilterVolumes(vm) + ExpectWithOffset(1, isVolumes).To(HaveLen(len(isStorage.Volumes))) + + for _, isVol := range isStorage.Volumes { + found := false + + for idx, vol := range isVolumes { + claim := vol.PersistentVolumeClaim.InstanceVolumeClaim + if claim.StorageClass == isStorage.StorageClass && claim.Size == isVol.Size { + isVolumes = append(isVolumes[:idx], isVolumes[idx+1:]...) + found = true + break + } + } + + ExpectWithOffset(1, found).To(BeTrue(), "failed to find instance storage volume for %v", isVol) + } + } + + BeforeEach(func() { + vmClass = builder.DummyVirtualMachineClassA2() + }) + + When("InstanceStorage FFS is enabled", func() { + + It("VM Class does not contain instance storage volumes", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeFalse()) + Expect(instancestorage.FilterVolumes(vmCtx.VM)).To(BeEmpty()) + }) + + When("Instance Volume is added in VM Class", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.InstanceStorage = builder.DummyInstanceStorageA2() + }) + + It("Instance Volumes should be added", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + expectInstanceStorageVolumes(vmCtx.VM, vmClass.Spec.Hardware.InstanceStorage) + }) + + It("Instance Storage is already added to VM Spec.Volumes", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + + isVolumesBefore := instancestorage.FilterVolumes(vmCtx.VM) + expectInstanceStorageVolumes(vmCtx.VM, vmClass.Spec.Hardware.InstanceStorage) + + // Instance Storage is already configured, should not patch again + is = vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + isVolumesAfter := instancestorage.FilterVolumes(vmCtx.VM) + Expect(isVolumesAfter).To(HaveLen(len(isVolumesBefore))) + Expect(isVolumesAfter).To(Equal(isVolumesBefore)) + }) + }) + }) + }) + + Context("HasPVC", func() { + + Context("Spec has no PVC", func() { + It("will return false", func() { + spec := vmopv1.VirtualMachineSpec{} + Expect(vsphere.HasPVC(spec)).To(BeFalse()) + }) + }) + + Context("Spec has PVCs", func() { + It("will return true", func() { + spec := vmopv1.VirtualMachineSpec{ + Volumes: []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + }, + } + Expect(vsphere.HasPVC(spec)).To(BeTrue()) + }) + }) + }) + + Context("HardwareVersionForPVCandPCIDevices", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + imageHWVersion int32 + ) + + BeforeEach(func() { + imageHWVersion = 14 + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + }) + + It("ConfigSpec has PCI devices and VM spec has PVCs", func() { + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(17))) + }) + + It("ConfigSpec has PCI devices and VM spec has no PVCs", func() { + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, false)).To(Equal(int32(17))) + }) + + It("ConfigSpec has PCI devices, VM spec has PVCs image hardware version is higher than min supported HW version for PCI devices", func() { + imageHWVersion = 18 + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(18))) + }) + + It("VM spec has PVCs and config spec has no devices", func() { + configSpec = &types.VirtualMachineConfigSpec{} + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(15))) + }) + + It("VM spec has PVCs, config spec has no devices and image hardware version is higher than min supported PVC HW version", func() { + configSpec = &types.VirtualMachineConfigSpec{} + imageHWVersion = 16 + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(16))) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go b/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go new file mode 100644 index 000000000..ac42dce8a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() + +func vcSimTests() { + Describe("CPUFreq", cpuFreqTests) + Describe("InitOvfCacheAndLockPool", initOvfCacheAndLockPoolTests) + Describe("ResourcePolicyTests", resourcePolicyTests) + Describe("VirtualMachine", vmTests) + Describe("VirtualMachineE2E", vmE2ETests) + Describe("VirtualMachineUtilsTest", vmUtilTests) +} + +func TestVSphereProvider(t *testing.T) { + suite.Register(t, "VMProvider Tests", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/test/builder/fake.go b/test/builder/fake.go index 7aab65f77..5764ceac1 100644 --- a/test/builder/fake.go +++ b/test/builder/fake.go @@ -42,13 +42,15 @@ func KnownObjectTypes() []client.Object { &v1alpha2.VirtualMachineService{}, &v1alpha1.VirtualMachineClass{}, &v1alpha2.VirtualMachineClass{}, - &cnsv1alpha1.CnsNodeVmAttachment{}, &v1alpha1.VirtualMachinePublishRequest{}, &v1alpha2.VirtualMachinePublishRequest{}, &v1alpha1.ClusterVirtualMachineImage{}, &v1alpha2.ClusterVirtualMachineImage{}, &v1alpha1.VirtualMachineImage{}, &v1alpha2.VirtualMachineImage{}, + &cnsv1alpha1.CnsNodeVmAttachment{}, + &ncpv1alpha1.VirtualNetworkInterface{}, + &netopv1alpha1.NetworkInterface{}, } } diff --git a/test/builder/utila2.go b/test/builder/utila2.go index a36c8bc11..902587367 100644 --- a/test/builder/utila2.go +++ b/test/builder/utila2.go @@ -14,100 +14,26 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ) -func DummyVirtualMachineSetResourcePolicyA2() *vmopv1.VirtualMachineSetResourcePolicy { - return &vmopv1.VirtualMachineSetResourcePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-", - }, - Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ - ResourcePool: vmopv1.ResourcePoolSpec{ - Name: "dummy-resource-pool", - Reservations: vmopv1.VirtualMachineResourceSpec{ - Cpu: resource.MustParse("1Gi"), - Memory: resource.MustParse("2Gi"), - }, - Limits: vmopv1.VirtualMachineResourceSpec{ - Cpu: resource.MustParse("2Gi"), - Memory: resource.MustParse("4Gi"), - }, - }, - Folder: "dummy-folder", - ClusterModuleGroups: []string{"dummy-cluster-modules"}, - }, - } -} - -func DummyVirtualMachineServiceA2() *vmopv1.VirtualMachineService { - return &vmopv1.VirtualMachineService{ +func DummyVirtualMachineClass2A2(name string) *vmopv1.VirtualMachineClass { + return &vmopv1.VirtualMachineClass{ ObjectMeta: metav1.ObjectMeta{ - // Using image.GenerateName causes problems with unit tests - Name: fmt.Sprintf("test-%s", uuid.New()), + Name: name, }, - Spec: vmopv1.VirtualMachineServiceSpec{ - Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, - Ports: []vmopv1.VirtualMachineServicePort{ - { - Name: "dummy-port", - Protocol: "TCP", - Port: 42, - TargetPort: 4242, - }, - }, - Selector: map[string]string{ - "foo": "bar", + Spec: vmopv1.VirtualMachineClassSpec{ + Hardware: vmopv1.VirtualMachineClassHardware{ + Cpus: int64(2), + Memory: resource.MustParse("4Gi"), }, - }, - } -} - -func DummyVirtualMachineA2() *vmopv1.VirtualMachine { - return &vmopv1.VirtualMachine{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-", - Labels: map[string]string{}, - Annotations: map[string]string{}, - }, - Spec: vmopv1.VirtualMachineSpec{ - ImageName: DummyImageName, - ClassName: DummyClassName, - PowerState: vmopv1.VirtualMachinePowerStateOn, - Volumes: []vmopv1.VirtualMachineVolume{ - { - Name: DummyVolumeName, - VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ - PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ - PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: DummyPVCName, - }, - }, + Policies: vmopv1.VirtualMachineClassPolicies{ + Resources: vmopv1.VirtualMachineClassResources{ + Requests: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("1Gi"), + Memory: resource.MustParse("2Gi"), + }, + Limits: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("2Gi"), + Memory: resource.MustParse("4Gi"), }, - }, - }, - }, - } -} - -func DummyVirtualMachinePublishRequestA2(name, namespace, sourceName, itemName, clName string) *vmopv1.VirtualMachinePublishRequest { - return &vmopv1.VirtualMachinePublishRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Finalizers: []string{"virtualmachinepublishrequest.vmoperator.vmware.com"}, - }, - Spec: vmopv1.VirtualMachinePublishRequestSpec{ - Source: vmopv1.VirtualMachinePublishRequestSource{ - Name: sourceName, - APIVersion: "vmoperator.vmware.com/v1alpha2", - Kind: "VirtualMachine", - }, - Target: vmopv1.VirtualMachinePublishRequestTarget{ - Item: vmopv1.VirtualMachinePublishRequestTargetItem{ - Name: itemName, - }, - Location: vmopv1.VirtualMachinePublishRequestTargetLocation{ - Name: clName, - APIVersion: "imageregistry.vmware.com/v1alpha1", - Kind: "ContentLibrary", }, }, }, @@ -140,6 +66,20 @@ func DummyVirtualMachineClassA2() *vmopv1.VirtualMachineClass { } } +func DummyInstanceStorageA2() vmopv1.InstanceStorage { + return vmopv1.InstanceStorage{ + StorageClass: DummyStorageClassName, + Volumes: []vmopv1.InstanceStorageVolume{ + { + Size: resource.MustParse("256Gi"), + }, + { + Size: resource.MustParse("512Gi"), + }, + }, + } +} + func DummyInstanceStorageVirtualMachineVolumesA2() []vmopv1.VirtualMachineVolume { return []vmopv1.VirtualMachineVolume{ { @@ -173,6 +113,161 @@ func DummyInstanceStorageVirtualMachineVolumesA2() []vmopv1.VirtualMachineVolume } } +func DummyBasicVirtualMachineA2(name, namespace string) *vmopv1.VirtualMachine { + return &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: vmopv1.VirtualMachineSpec{ + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePowerStateOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + }, + } +} + +func DummyVirtualMachineA2() *vmopv1.VirtualMachine { + return &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: vmopv1.VirtualMachineSpec{ + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePowerStateOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + Volumes: []vmopv1.VirtualMachineVolume{ + { + Name: DummyVolumeName, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: DummyPVCName, + }, + }, + }, + }, + }, + /* TODO: Convert this if/as needed + NetworkInterfaces: []vmopv1.VirtualMachineNetworkInterface{ + { + NetworkName: DummyNetworkName, + NetworkType: "", + }, + { + NetworkName: DummyNetworkName + "-2", + NetworkType: "", + }, + }, + VmMetadata: &vmopv1.VirtualMachineMetadata{ + ConfigMapName: DummyMetadataCMName, + Transport: "ExtraConfig", + }, + */ + }, + } +} + +func AddDummyInstanceStorageVolumeA2(vm *vmopv1.VirtualMachine) { + vm.Spec.Volumes = append(vm.Spec.Volumes, DummyInstanceStorageVirtualMachineVolumesA2()...) +} + +func DummyVirtualMachineServiceA2() *vmopv1.VirtualMachineService { + return &vmopv1.VirtualMachineService{ + ObjectMeta: metav1.ObjectMeta{ + // Using image.GenerateName causes problems with unit tests + Name: fmt.Sprintf("test-%s", uuid.New()), + }, + Spec: vmopv1.VirtualMachineServiceSpec{ + Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, + Ports: []vmopv1.VirtualMachineServicePort{ + { + Name: "dummy-port", + Protocol: "TCP", + Port: 42, + TargetPort: 4242, + }, + }, + Selector: map[string]string{ + "foo": "bar", + }, + }, + } +} + +func DummyVirtualMachineSetResourcePolicyA2() *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: "dummy-resource-pool", + Reservations: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("1Gi"), + Memory: resource.MustParse("2Gi"), + }, + Limits: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("2Gi"), + Memory: resource.MustParse("4Gi"), + }, + }, + Folder: "dummy-folder", + ClusterModuleGroups: []string{"dummy-cluster-modules"}, + }, + } +} + +func DummyVirtualMachineSetResourcePolicy2A2(name, namespace string) *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: name, + }, + Folder: name, + }, + } +} + +func DummyVirtualMachinePublishRequestA2(name, namespace, sourceName, itemName, clName string) *vmopv1.VirtualMachinePublishRequest { + return &vmopv1.VirtualMachinePublishRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Finalizers: []string{"virtualmachinepublishrequest.vmoperator.vmware.com"}, + }, + Spec: vmopv1.VirtualMachinePublishRequestSpec{ + Source: vmopv1.VirtualMachinePublishRequestSource{ + Name: sourceName, + APIVersion: "vmoperator.vmware.com/v1alpha2", + Kind: "VirtualMachine", + }, + Target: vmopv1.VirtualMachinePublishRequestTarget{ + Item: vmopv1.VirtualMachinePublishRequestTargetItem{ + Name: itemName, + }, + Location: vmopv1.VirtualMachinePublishRequestTargetLocation{ + Name: clName, + APIVersion: "imageregistry.vmware.com/v1alpha1", + Kind: "ContentLibrary", + }, + }, + }, + } +} + func DummyVirtualMachineImageA2(imageName string) *vmopv1.VirtualMachineImage { return &vmopv1.VirtualMachineImage{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/builder/vcsim_test_context.go b/test/builder/vcsim_test_context.go index 9805a311f..daad84acc 100644 --- a/test/builder/vcsim_test_context.go +++ b/test/builder/vcsim_test_context.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Package builder is a comment just to silence the linter @@ -45,8 +45,11 @@ import ( _ "github.com/vmware/govmomi/vapi/cluster/simulator" _ "github.com/vmware/govmomi/vapi/simulator" - vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/conditions2" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/record" "github.com/vmware-tanzu/vm-operator/test/testutil" @@ -55,8 +58,9 @@ import ( type NetworkEnv string const ( - NetworkEnvVDS = NetworkEnv("vds") - NetworkEnvNSXT = NetworkEnv("nsx-t") + NetworkEnvVDS = NetworkEnv("vds") + NetworkEnvNSXT = NetworkEnv("nsx-t") + NetworkEnvNamed = NetworkEnv("named") NsxTLogicalSwitchUUID = "nsxt-dummy-ls-uuid" ) @@ -96,6 +100,9 @@ type VCSimTestConfig struct { // WithVMClassAsConfigDaynDate enables the WCP_VM_CLASS_AS_CONFIG_DAYNDATE FSS. WithVMClassAsConfigDaynDate bool + // WithV1A2 enables the VMServiceV1Alpha2FSS FSS. + WithV1A2 bool + // WithNetworkEnv is the network environment type. WithNetworkEnv NetworkEnv } @@ -137,6 +144,7 @@ type TestContextForVCSim struct { folder *object.Folder datastore *object.Datastore withFaultDomains bool + withV1A2 bool singleCCR *object.ClusterComputeResource azCCRs map[string][]*object.ClusterComputeResource @@ -160,7 +168,7 @@ func (s *TestSuite) NewTestContextForVCSim( ctx := newTestContextForVCSim(config, initObjects) - ctx.setupEnvFSS(config) + ctx.setupEnv(config) ctx.setupVCSim(config) ctx.setupContentLibrary(config) ctx.setupK8sConfig(config) @@ -180,6 +188,7 @@ func newTestContextForVCSim( PodNamespace: "vmop-pod-test", Recorder: fakeRecorder, withFaultDomains: config.WithFaultDomains, + withV1A2: config.WithV1A2, } if ctx.withFaultDomains { @@ -272,19 +281,22 @@ func (c *TestContextForVCSim) CreateWorkloadNamespace() WorkloadNamespaceInfo { Expect(c.Client.Update(c, ns)).To(Succeed()) } - if clID := c.ContentLibraryID; clID != "" { - csBinding := &vmopv1.ContentSourceBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - Namespace: ns.Name, - }, - ContentSourceRef: vmopv1.ContentSourceReference{ - APIVersion: vmopv1.SchemeGroupVersion.Group, - Kind: "ContentSource", - Name: clID, - }, + // Not the exact right FFS, but it's what we've plumbed and is otherwise implied. + if !c.withV1A2 { + if clID := c.ContentLibraryID; clID != "" { + csBinding := &v1alpha1.ContentSourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + Namespace: ns.Name, + }, + ContentSourceRef: v1alpha1.ContentSourceReference{ + APIVersion: v1alpha1.SchemeGroupVersion.Group, + Kind: "ContentSource", + Name: clID, + }, + } + Expect(c.Client.Create(c, csBinding)).To(Succeed()) } - Expect(c.Client.Create(c, csBinding)).To(Succeed()) } resourceQuota := &corev1.ResourceQuota{ @@ -313,10 +325,26 @@ func (c *TestContextForVCSim) CreateWorkloadNamespace() WorkloadNamespaceInfo { } } -// TODO: Get rid of runtime env checks so this isn't needed. -func (c *TestContextForVCSim) setupEnvFSS(config VCSimTestConfig) { +func (c *TestContextForVCSim) setupEnv(config VCSimTestConfig) { Expect(lib.SetVMOpNamespaceEnv(c.PodNamespace)).To(Succeed()) + switch config.WithNetworkEnv { + case NetworkEnvVDS: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeVDS)).To(Succeed()) + case NetworkEnvNSXT: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNSXT)).To(Succeed()) + case NetworkEnvNamed: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNamed)).To(Succeed()) + default: + Expect(os.Unsetenv(lib.NetworkProviderType)).To(Succeed()) + } + + v1a2 := "false" + if config.WithV1A2 { + v1a2 = "true" + } + Expect(os.Setenv(lib.VMServiceV1Alpha2FSS, v1a2)).To(Succeed()) + if config.WithContentLibrary { Expect(os.Setenv("CONTENT_API_WAIT_SECS", "1")).To(Succeed()) } @@ -465,32 +493,6 @@ func (c *TestContextForVCSim) setupContentLibrary(config VCSimTestConfig) { Expect(clID).ToNot(BeEmpty()) c.ContentLibraryID = clID - clProvider := &vmopv1.ContentLibraryProvider{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - }, - Spec: vmopv1.ContentLibraryProviderSpec{ - UUID: clID, - }, - } - Expect(c.Client.Create(c, clProvider)).To(Succeed()) - - cs := &vmopv1.ContentSource{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - }, - Spec: vmopv1.ContentSourceSpec{ - ProviderRef: vmopv1.ContentProviderReference{ - Name: clProvider.Name, - Kind: "ContentLibraryProvider", - }, - }, - } - Expect(c.Client.Create(c, cs)).To(Succeed()) - - Expect(controllerutil.SetOwnerReference(cs, clProvider, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Update(c, clProvider)).To(Succeed()) - libraryItem := library.Item{ Name: "test-image-ovf", Type: "ovf", @@ -498,12 +500,50 @@ func (c *TestContextForVCSim) setupContentLibrary(config VCSimTestConfig) { } c.ContentLibraryImageName = libraryItem.Name - vmImage := DummyVirtualMachineImage(c.ContentLibraryImageName) - Expect(controllerutil.SetOwnerReference(clProvider, vmImage, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Create(c, vmImage)).To(Succeed()) - - createContentLibraryItem(libMgr, libraryItem, + itemID := createContentLibraryItem(libMgr, libraryItem, path.Join(testutil.GetRootDirOrDie(), "images", "ttylinux-pc_i486-16.1.ovf")) + + // Not the exact right FFS, but it's what we've plumbed and is otherwise implied. + if c.withV1A2 { + // The image isn't quite as prod but sufficient for what we need here ATM. + clusterVMImage := DummyClusterVirtualMachineImageA2(c.ContentLibraryImageName) + clusterVMImage.Spec.ProviderRef.Kind = "ClusterContentLibraryItem" + Expect(c.Client.Create(c, clusterVMImage)).To(Succeed()) + clusterVMImage.Status.ProviderItemID = itemID + conditions2.MarkTrue(clusterVMImage, v1alpha2.VirtualMachineImageSyncedCondition) // TODO: Until we get rollup Ready condition + Expect(c.Client.Status().Update(c, clusterVMImage)).To(Succeed()) + + } else { + clProvider := &v1alpha1.ContentLibraryProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + }, + Spec: v1alpha1.ContentLibraryProviderSpec{ + UUID: clID, + }, + } + Expect(c.Client.Create(c, clProvider)).To(Succeed()) + + cs := &v1alpha1.ContentSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + }, + Spec: v1alpha1.ContentSourceSpec{ + ProviderRef: v1alpha1.ContentProviderReference{ + Name: clProvider.Name, + Kind: "ContentLibraryProvider", + }, + }, + } + Expect(c.Client.Create(c, cs)).To(Succeed()) + + Expect(controllerutil.SetOwnerReference(cs, clProvider, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Update(c, clProvider)).To(Succeed()) + + vmImage := DummyVirtualMachineImage(c.ContentLibraryImageName) + Expect(controllerutil.SetOwnerReference(clProvider, vmImage, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Create(c, vmImage)).To(Succeed()) + } } func (c *TestContextForVCSim) ContentLibraryItemTemplate(srcVMName, templateName string) { @@ -529,21 +569,30 @@ func (c *TestContextForVCSim) ContentLibraryItemTemplate(srcVMName, templateName }, } - _, err = vcenter.NewManager(c.RestClient).CreateTemplate(c, spec) + itemID, err := vcenter.NewManager(c.RestClient).CreateTemplate(c, spec) Expect(err).ToNot(HaveOccurred()) // Create the expected VirtualMachineImage for the template. - vmImage := DummyVirtualMachineImage(templateName) - cl := &vmopv1.ContentLibraryProvider{} - Expect(c.Client.Get(c, client.ObjectKey{Name: clID}, cl)).To(Succeed()) - Expect(controllerutil.SetOwnerReference(cl, vmImage, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Create(c, vmImage)).To(Succeed()) + if c.withV1A2 { + clusterVMImage := DummyClusterVirtualMachineImageA2(templateName) + clusterVMImage.Spec.ProviderRef.Kind = "ClusterContentLibraryItem" + Expect(c.Client.Create(c, clusterVMImage)).To(Succeed()) + clusterVMImage.Status.ProviderItemID = itemID + conditions2.MarkTrue(clusterVMImage, v1alpha2.VirtualMachineImageSyncedCondition) // TODO: Until we get rollup Ready condition + Expect(c.Client.Status().Update(c, clusterVMImage)).To(Succeed()) + } else { + vmImage := DummyVirtualMachineImage(templateName) + cl := &v1alpha1.ContentLibraryProvider{} + Expect(c.Client.Get(c, client.ObjectKey{Name: clID}, cl)).To(Succeed()) + Expect(controllerutil.SetOwnerReference(cl, vmImage, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Create(c, vmImage)).To(Succeed()) + } } func createContentLibraryItem( libMgr *library.Manager, libraryItem library.Item, - itemPath string) { + itemPath string) string { ctx := goctx.Background() @@ -590,6 +639,8 @@ func createContentLibraryItem( } Expect(uploadFunc(itemPath)).To(Succeed()) Expect(libMgr.CompleteLibraryItemUpdateSession(ctx, sessionID)).To(Succeed()) + + return itemID } func (c *TestContextForVCSim) setupK8sConfig(config VCSimTestConfig) { @@ -720,11 +771,42 @@ func (c *TestContextForVCSim) GetAZClusterComputes(azName string) []*object.Clus func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicy( name string, - nsInfo WorkloadNamespaceInfo) (*vmopv1.VirtualMachineSetResourcePolicy, *object.Folder) { + nsInfo WorkloadNamespaceInfo) (*v1alpha1.VirtualMachineSetResourcePolicy, *object.Folder) { + + ExpectWithOffset(1, c.withV1A2).To(BeFalse()) resourcePolicy := DummyVirtualMachineSetResourcePolicy2(name, nsInfo.Namespace) Expect(c.Client.Create(c, resourcePolicy)).To(Succeed()) + folder := c.createVirtualMachineSetResourcePolicyCommon( + resourcePolicy.Spec.ResourcePool.Name, + resourcePolicy.Spec.Folder.Name, + nsInfo) + + return resourcePolicy, folder +} + +func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicyA2( + name string, + nsInfo WorkloadNamespaceInfo) (*v1alpha2.VirtualMachineSetResourcePolicy, *object.Folder) { + + ExpectWithOffset(1, c.withV1A2).To(BeTrue()) + + resourcePolicy := DummyVirtualMachineSetResourcePolicy2A2(name, nsInfo.Namespace) + Expect(c.Client.Create(c, resourcePolicy)).To(Succeed()) + + folder := c.createVirtualMachineSetResourcePolicyCommon( + resourcePolicy.Spec.ResourcePool.Name, + resourcePolicy.Spec.Folder, + nsInfo) + + return resourcePolicy, folder +} + +func (c *TestContextForVCSim) createVirtualMachineSetResourcePolicyCommon( + rpName, folderName string, + nsInfo WorkloadNamespaceInfo) *object.Folder { + var rps []*object.ResourcePool if c.withFaultDomains { @@ -749,14 +831,14 @@ func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicy( nsRP, ok := objRef.(*object.ResourcePool) Expect(ok).To(BeTrue()) - _, err = nsRP.Create(c, resourcePolicy.Spec.ResourcePool.Name, types.DefaultResourceConfigSpec()) + _, err = nsRP.Create(c, rpName, types.DefaultResourceConfigSpec()) Expect(err).ToNot(HaveOccurred()) } - folder, err := nsInfo.Folder.CreateFolder(c, resourcePolicy.Spec.Folder.Name) + folder, err := nsInfo.Folder.CreateFolder(c, folderName) Expect(err).ToNot(HaveOccurred()) - return resourcePolicy, folder + return folder } func (c *TestContextForVCSim) GetVMFromMoID(moID string) *object.VirtualMachine { diff --git a/webhooks/virtualmachine/webhooks.go b/webhooks/virtualmachine/webhooks.go index 49bf76683..83370a430 100644 --- a/webhooks/virtualmachine/webhooks.go +++ b/webhooks/virtualmachine/webhooks.go @@ -9,12 +9,21 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := v1alpha1.AddToManager(ctx, mgr); err != nil { - return errors.Wrap(err, "failed to initialize v1alpha1 webhooks") + if lib.IsVMServiceV1Alpha2FSSEnabled() { + // TODO: We'll likely still need the v1a1 mutation wehbook (at least some limited version of it) + if err := v1alpha2.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize v1alpha2 webhooks") + } + } else { + if err := v1alpha1.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize v1alpha1 webhooks") + } } return nil diff --git a/webhooks/virtualmachineclass/v1alpha2/webhooks.go b/webhooks/virtualmachineclass/v1alpha2/webhooks.go index f727028ec..226641b85 100644 --- a/webhooks/virtualmachineclass/v1alpha2/webhooks.go +++ b/webhooks/virtualmachineclass/v1alpha2/webhooks.go @@ -1,7 +1,7 @@ // Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package virtualmachineclass +package v1alpha2 import ( "github.com/pkg/errors" diff --git a/webhooks/virtualmachineclass/webhooks.go b/webhooks/virtualmachineclass/webhooks.go index 9f6063a6c..4611b5248 100644 --- a/webhooks/virtualmachineclass/webhooks.go +++ b/webhooks/virtualmachineclass/webhooks.go @@ -7,9 +7,15 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineclass/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineclass/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } + return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go b/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go index 213223611..8c2581a5a 100644 --- a/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go +++ b/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go @@ -1,7 +1,7 @@ // Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package virtualmachinepublishrequest +package v1alpha2 import ( "github.com/pkg/errors" diff --git a/webhooks/virtualmachinepublishrequest/webhooks.go b/webhooks/virtualmachinepublishrequest/webhooks.go index 5c3759fce..91a84119f 100644 --- a/webhooks/virtualmachinepublishrequest/webhooks.go +++ b/webhooks/virtualmachinepublishrequest/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinepublishrequest/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinepublishrequest/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachineservice/webhooks.go b/webhooks/virtualmachineservice/webhooks.go index 440b05332..c13880752 100644 --- a/webhooks/virtualmachineservice/webhooks.go +++ b/webhooks/virtualmachineservice/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineservice/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineservice/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachinesetresourcepolicy/webhooks.go b/webhooks/virtualmachinesetresourcepolicy/webhooks.go index 6e064b195..8d123c0bf 100644 --- a/webhooks/virtualmachinesetresourcepolicy/webhooks.go +++ b/webhooks/virtualmachinesetresourcepolicy/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinesetresourcepolicy/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinesetresourcepolicy/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) }