From eb9ac27cee41672fa02013d4ff96688abdcb2acb Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 30 Aug 2023 13:55:55 -0500 Subject: [PATCH 1/5] Add AppendNewExtraConfigValues() --- pkg/util/configspec.go | 23 +++++++++++++++++++++++ pkg/util/configspec_test.go | 22 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/pkg/util/configspec.go b/pkg/util/configspec.go index 3d070bad6..97d07d444 100644 --- a/pkg/util/configspec.go +++ b/pkg/util/configspec.go @@ -165,3 +165,26 @@ func RemoveDevicesFromConfigSpec(configSpec *vimTypes.VirtualMachineConfigSpec, } configSpec.DeviceChange = targetDevChanges } + +// AppendNewExtraConfigValues add the new extra config values if not already present in the extra config. +func AppendNewExtraConfigValues( + extraConfig []vimTypes.BaseOptionValue, + newECMap map[string]string) []vimTypes.BaseOptionValue { + + ecMap := make(map[string]vimTypes.AnyType) + for _, opt := range extraConfig { + if optValue := opt.GetOptionValue(); optValue != nil { + ecMap[optValue.Key] = optValue.Value + } + } + + // Only add fields that aren't already in the ExtraConfig. + var newExtraConfig []vimTypes.BaseOptionValue + for k, v := range newECMap { + if _, exists := ecMap[k]; !exists { + newExtraConfig = append(newExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + + return append(extraConfig, newExtraConfig...) +} diff --git a/pkg/util/configspec_test.go b/pkg/util/configspec_test.go index b5cdf2d20..f53594152 100644 --- a/pkg/util/configspec_test.go +++ b/pkg/util/configspec_test.go @@ -214,6 +214,28 @@ var _ = Describe("RemoveDevicesFromConfigSpec", func() { }) }) +var _ = Describe("AppendNewExtraConfigValues", func() { + + It("only adds new values not already in the ExtraConfig", func() { + ec := []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{ + Key: "key1", + Value: "keep-me", + }, + } + + newECMap := map[string]string{ + "key1": "should-be-ignored", + "key2": "add-me", + } + + newExtraConfig := util.AppendNewExtraConfigValues(ec, newECMap) + Expect(newExtraConfig).To(HaveLen(2)) + Expect(newExtraConfig).To(ContainElement(&vimTypes.OptionValue{Key: "key1", Value: "keep-me"})) + Expect(newExtraConfig).To(ContainElement(&vimTypes.OptionValue{Key: "key2", Value: "add-me"})) + }) +}) + var _ = Describe("SanitizeVMClassConfigSpec", func() { oldVMClassAsConfigFSSEnabledFunc := lib.IsVMClassAsConfigFSSEnabled var ( From 517a81aba800ea21c7b4266a58a669216382f922 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 30 Aug 2023 14:02:54 -0500 Subject: [PATCH 2/5] Add GetNetworkProviderType() --- pkg/lib/env.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/lib/env.go b/pkg/lib/env.go index 06591194e..68059b2c0 100644 --- a/pkg/lib/env.go +++ b/pkg/lib/env.go @@ -85,7 +85,11 @@ func GetVMOpNamespaceFromEnv() (string, error) { } var IsNamedNetworkProviderEnabled = func() bool { - return os.Getenv(NetworkProviderType) == NetworkProviderTypeNamed + return GetNetworkProviderType() == NetworkProviderTypeNamed +} + +func GetNetworkProviderType() string { + return os.Getenv(NetworkProviderType) } var IsWcpFaultDomainsFSSEnabled = func() bool { From 57e338042414f8d7bdc1027b5eb83ad2506bd231 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 30 Aug 2023 14:06:53 -0500 Subject: [PATCH 3/5] Initial v1a2 provider There is still a lot of work left to do before switching over to v1a2 but this is able to create TKGs with everything else still only understanding v1a1, so we have a good baseline of version conversion and functionality. Unfortunately the way the existing code is and our need to support the existing v1a1 and new v1a2 APIs in the same codebase makes this a large and painful change. The goal is to wrap up the remaining items as quickly as possible so we can move to v1a2 and then really start to focus on some of our long term tech debt so future version bumps aren't nearly as painful. --- api/v1alpha1/condition_conversion.go | 6 + api/v1alpha1/virtualmachine_conversion.go | 28 +- pkg/conditions2/setter.go | 21 +- pkg/conditions2/setter_test.go | 2 +- pkg/manager/manager.go | 8 +- .../providers/vsphere/vmprovider_vm_test.go | 3 + .../providers/vsphere2/client/client.go | 255 +++ .../vsphere2/client/client_suite_test.go | 46 + .../providers/vsphere2/client/client_test.go | 366 ++++ .../clustermodules/cluster_modules.go | 8 + .../cluster_modules_provider.go | 127 ++ .../cluster_modules_suite_test.go | 26 + .../clustermodules/cluster_modules_test.go | 103 + .../clustermodules/cluster_modules_utils.go | 62 + .../cluster_modules_utils_test.go | 153 ++ .../providers/vsphere2/config/config.go | 290 +++ .../vsphere2/config/config_suite_test.go | 26 + .../providers/vsphere2/config/config_test.go | 211 ++ .../providers/vsphere2/constants/constants.go | 137 ++ .../contentlibrary/content_library.go | 10 + .../content_library_provider.go | 379 ++++ .../content_library_suite_test.go | 26 + .../contentlibrary/content_library_test.go | 139 ++ .../contentlibrary/content_library_utils.go | 149 ++ .../content_library_utils_test.go | 28 + .../vsphere2/credentials/credentials.go | 62 + .../credentials/credentials_suite_test.go | 44 + .../vsphere2/credentials/credentials_test.go | 68 + .../instancestorage/instance_storage.go | 41 + .../providers/vsphere2/internal/internal.go | 59 + .../providers/vsphere2/network/gosc.go | 79 + .../providers/vsphere2/network/gosc_test.go | 132 ++ .../providers/vsphere2/network/netplan.go | 103 + .../vsphere2/network/netplan_test.go | 143 ++ .../providers/vsphere2/network/network.go | 630 ++++++ .../vsphere2/network/network_suite_test.go | 22 + .../vsphere2/network/network_test.go | 344 ++++ .../providers/vsphere2/network/nsxt.go | 117 ++ .../providers/vsphere2/network/nsxt_test.go | 71 + .../vsphere2/placement/cluster_placement.go | 206 ++ .../placement/cluster_placement_test.go | 136 ++ .../placement/placement_suite_test.go | 26 + .../vsphere2/placement/zone_placement.go | 333 +++ .../vsphere2/placement/zone_placement_test.go | 284 +++ .../providers/vsphere2/resources/vm.go | 230 +++ .../providers/vsphere2/session/session.go | 41 + .../vsphere2/session/session_suite_test.go | 22 + .../vsphere2/session/session_util.go | 34 + .../vsphere2/session/session_util_test.go | 93 + .../providers/vsphere2/session/session_vm.go | 60 + .../vsphere2/session/session_vm_update.go | 957 +++++++++ .../session/session_vm_update_test.go | 1236 +++++++++++ .../vsphere2/storage/provisioning.go | 88 + .../vsphere2/storage/storageclass.go | 83 + pkg/vmprovider/providers/vsphere2/test/pki.go | 70 + .../providers/vsphere2/test/suite.go | 61 + .../providers/vsphere2/test/vcsim.go | 31 + .../providers/vsphere2/vcenter/cluster.go | 44 + .../vsphere2/vcenter/cluster_test.go | 47 + .../providers/vsphere2/vcenter/folder.go | 138 ++ .../providers/vsphere2/vcenter/folder_test.go | 173 ++ .../providers/vsphere2/vcenter/getvm.go | 152 ++ .../providers/vsphere2/vcenter/getvm_test.go | 158 ++ .../providers/vsphere2/vcenter/host.go | 41 + .../providers/vsphere2/vcenter/host_test.go | 66 + .../vsphere2/vcenter/resourcepool.go | 163 ++ .../vsphere2/vcenter/resourcepool_test.go | 200 ++ .../vsphere2/vcenter/vcenter_suite_test.go | 30 + .../providers/vsphere2/virtualmachine/ccr.go | 34 + .../vsphere2/virtualmachine/ccr_test.go | 42 + .../vsphere2/virtualmachine/configspec.go | 190 ++ .../virtualmachine/configspec_test.go | 278 +++ .../vsphere2/virtualmachine/conversion.go | 18 + .../virtualmachine/conversion_test.go | 34 + .../vsphere2/virtualmachine/delete.go | 44 + .../vsphere2/virtualmachine/delete_test.go | 71 + .../vsphere2/virtualmachine/devices.go | 100 + .../vsphere2/virtualmachine/heartbeat.go | 25 + .../vsphere2/virtualmachine/publish.go | 64 + .../vsphere2/virtualmachine/publish_test.go | 98 + .../vsphere2/virtualmachine/storage.go | 41 + .../virtualmachine_suite_test.go | 28 + .../virtualmachine/webconsole_ticket.go | 65 + .../virtualmachine/webconsole_ticket_test.go | 43 + .../vsphere2/vmlifecycle/bootstrap.go | 265 +++ .../vmlifecycle/bootstrap_cloudinit.go | 186 ++ .../vmlifecycle/bootstrap_cloudinit_test.go | 398 ++++ .../vmlifecycle/bootstrap_linuxprep.go | 55 + .../vmlifecycle/bootstrap_linuxprep_test.go | 156 ++ .../vsphere2/vmlifecycle/bootstrap_sysprep.go | 79 + .../vmlifecycle/bootstrap_sysprep_test.go | 153 ++ .../vmlifecycle/bootstrap_templatedata.go | 453 ++++ .../bootstrap_templatedata_test.go | 221 ++ .../vsphere2/vmlifecycle/bootstrap_test.go | 51 + .../vmlifecycle/bootstrap_vappconfig.go | 108 + .../vmlifecycle/bootstrap_vappconfig_test.go | 267 +++ .../providers/vsphere2/vmlifecycle/create.go | 41 + .../vsphere2/vmlifecycle/create_clone.go | 188 ++ .../vmlifecycle/create_contentlibrary.go | 105 + .../vsphere2/vmlifecycle/update_status.go | 345 ++++ .../vmlifecycle/update_status_test.go | 211 ++ .../vmlifecycle/vmlifecycle_suite_test.go | 22 + .../providers/vsphere2/vmprovider.go | 428 ++++ .../vsphere2/vmprovider_resourcepolicy.go | 261 +++ .../vmprovider_resourcepolicy_test.go | 234 +++ .../providers/vsphere2/vmprovider_test.go | 88 + .../providers/vsphere2/vmprovider_vm.go | 1169 +++++++++++ .../providers/vsphere2/vmprovider_vm2_test.go | 256 +++ .../providers/vsphere2/vmprovider_vm_test.go | 1813 +++++++++++++++++ .../providers/vsphere2/vmprovider_vm_utils.go | 335 +++ .../vsphere2/vmprovider_vm_utils_test.go | 629 ++++++ .../providers/vsphere2/vsphere_suite_test.go | 31 + test/builder/fake.go | 4 +- test/builder/utila2.go | 275 ++- test/builder/vcsim_test_context.go | 204 +- 115 files changed, 19769 insertions(+), 164 deletions(-) create mode 100644 pkg/vmprovider/providers/vsphere2/client/client.go create mode 100644 pkg/vmprovider/providers/vsphere2/client/client_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/client/client_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/config/config.go create mode 100644 pkg/vmprovider/providers/vsphere2/config/config_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/config/config_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/constants/constants.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/credentials/credentials.go create mode 100644 pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go create mode 100644 pkg/vmprovider/providers/vsphere2/internal/internal.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/gosc.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/gosc_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/netplan.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/netplan_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/network.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/network_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/network_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/nsxt.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/nsxt_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/zone_placement.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/resources/vm.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_util.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_util_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_vm.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_vm_update.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/storage/provisioning.go create mode 100644 pkg/vmprovider/providers/vsphere2/storage/storageclass.go create mode 100644 pkg/vmprovider/providers/vsphere2/test/pki.go create mode 100644 pkg/vmprovider/providers/vsphere2/test/suite.go create mode 100644 pkg/vmprovider/providers/vsphere2/test/vcsim.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/cluster.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/folder.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/getvm.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/host.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/host_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go diff --git a/api/v1alpha1/condition_conversion.go b/api/v1alpha1/condition_conversion.go index 3211b0c24..275ef4410 100644 --- a/api/v1alpha1/condition_conversion.go +++ b/api/v1alpha1/condition_conversion.go @@ -17,6 +17,12 @@ func Convert_v1alpha1_Condition_To_v1_Condition(in *Condition, out *metav1.Condi out.Reason = in.Reason out.Message = in.Message + // The metav1.Condition requires the reason to be non-empty, when it was not in our prior v1a1 Condition. + // We don't have any great options as to what we can fill this in as. + if out.Reason == "" { + out.Reason = string(out.Status) + } + // TODO: out.ObservedGeneration = return nil diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index 905fb8405..536695bd3 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -155,7 +155,7 @@ func convert_v1alpha1_VmMetadata_To_v1alpha2_BootstrapSpec( out.CloudInit = &v1alpha2.VirtualMachineBootstrapCloudInitSpec{ RawCloudConfig: corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: objectName}, - Key: "guestinfo.userdata", + Key: "guestinfo.userdata", // TODO: Is this good enough? v1a1 would include everything with the "guestinfo" prefix }, } case VirtualMachineMetadataOvfEnvTransport: @@ -449,6 +449,12 @@ func convert_v1alpha2_NetworkStatus_To_v1alpha1_Network( func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( in *VirtualMachineSpec, out *v1alpha2.VirtualMachineSpec, s apiconversion.Scope) error { + // The generated auto convert will convert the power modes as-is strings which breaks things, so keep + // this first. + if err := autoConvert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha1_VirtualMachinePowerState_To_v1alpha2_VirtualMachinePowerState(in.PowerState) out.PowerOffMode = convert_v1alpha1_VirtualMachinePowerOpMode_To_v1alpha2_VirtualMachinePowerOpMode(in.PowerOffMode) out.SuspendMode = convert_v1alpha1_VirtualMachinePowerOpMode_To_v1alpha2_VirtualMachinePowerOpMode(in.SuspendMode) @@ -469,12 +475,16 @@ func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( // Deprecated: // in.Ports - return autoConvert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec(in, out, s) + return nil } func Convert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec( in *v1alpha2.VirtualMachineSpec, out *VirtualMachineSpec, s apiconversion.Scope) error { + if err := autoConvert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha2_VirtualMachinePowerState_To_v1alpha1_VirtualMachinePowerState(in.PowerState) out.PowerOffMode = convert_v1alpha2_VirtualMachinePowerOpMode_To_v1alpha1_VirtualMachinePowerOpMode(in.PowerOffMode) out.SuspendMode = convert_v1alpha2_VirtualMachinePowerOpMode_To_v1alpha1_VirtualMachinePowerOpMode(in.SuspendMode) @@ -496,7 +506,7 @@ func Convert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec( // Deprecated: // out.Ports - return autoConvert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec(in, out, s) + return nil } func Convert_v1alpha1_VirtualMachineVolumeStatus_To_v1alpha2_VirtualMachineVolumeStatus( @@ -518,18 +528,26 @@ func Convert_v1alpha2_VirtualMachineVolumeStatus_To_v1alpha1_VirtualMachineVolum func Convert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus( in *VirtualMachineStatus, out *v1alpha2.VirtualMachineStatus, s apiconversion.Scope) error { + if err := autoConvert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha1_VirtualMachinePowerState_To_v1alpha2_VirtualMachinePowerState(in.PowerState) out.Network = convert_v1alpha1_Network_To_v1alpha2_NetworkStatus(in.VmIp, in.NetworkInterfaces) out.LastRestartTime = in.LastRestartTime // WARNING: in.Phase requires manual conversion: does not exist in peer-type - return autoConvert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus(in, out, s) + return nil } func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( in *v1alpha2.VirtualMachineStatus, out *VirtualMachineStatus, s apiconversion.Scope) error { + if err := autoConvert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha2_VirtualMachinePowerState_To_v1alpha1_VirtualMachinePowerState(in.PowerState) out.Phase = convert_v1alpha2_Conditions_To_v1alpha1_Phase(in.Conditions) out.VmIp, out.NetworkInterfaces = convert_v1alpha2_NetworkStatus_To_v1alpha1_Network(in.Network) @@ -538,7 +556,7 @@ func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( // WARNING: in.Image requires manual conversion: does not exist in peer-type // WARNING: in.Class requires manual conversion: does not exist in peer-type - return autoConvert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus(in, out, s) + return nil } // ConvertTo converts this VirtualMachine to the Hub version. diff --git a/pkg/conditions2/setter.go b/pkg/conditions2/setter.go index b6f6520dc..0194fe577 100644 --- a/pkg/conditions2/setter.go +++ b/pkg/conditions2/setter.go @@ -78,13 +78,20 @@ func Set(to Setter, condition *metav1.Condition) { func TrueCondition(t string) *metav1.Condition { return &metav1.Condition{ Type: t, - Reason: t, // BMV: This is required field in metav1.Conditions. Fixup API later. Status: metav1.ConditionTrue, + // This is a non-empty field in metav1.Conditions, when it was not in our v1a1 Conditions. This + // really doesn't work with how we've defined our conditions so do something to make things + // work for now. + Reason: string(metav1.ConditionTrue), } } // FalseCondition returns a condition with Status=False and the given type. func FalseCondition(t string, reason string, messageFormat string, messageArgs ...interface{}) *metav1.Condition { + if reason == "" { + reason = string(metav1.ConditionFalse) + } + return &metav1.Condition{ Type: t, Status: metav1.ConditionFalse, @@ -95,6 +102,10 @@ func FalseCondition(t string, reason string, messageFormat string, messageArgs . // UnknownCondition returns a condition with Status=Unknown and the given type. func UnknownCondition(t string, reason string, messageFormat string, messageArgs ...interface{}) *metav1.Condition { + if reason == "" { + reason = string(metav1.ConditionUnknown) + } + return &metav1.Condition{ Type: t, Status: metav1.ConditionUnknown, @@ -124,14 +135,14 @@ func SetSummary(to Setter, options ...MergeOption) { Set(to, summary(to, options...)) } -// SetMirror creates a new condition by mirroring the the Ready condition from a dependent object; -// if the Ready condition does not exists in the source object, no target conditions is generated. +// SetMirror creates a new condition by mirroring the Ready condition from a dependent object; +// if the Ready condition does not exist in the source object, no target conditions is generated. func SetMirror(to Setter, targetCondition string, from Getter, options ...MirrorOptions) { Set(to, mirror(from, targetCondition, options...)) } -// SetAggregate creates a new condition with the aggregation of all the the Ready condition -// from a list of dependent objects; if the Ready condition does not exists in one of the source object, +// SetAggregate creates a new condition with the aggregation of all the Ready condition +// from a list of dependent objects; if the Ready condition does not exist in one of the source object, // the object is excluded from the aggregation; if none of the source object have ready condition, // no target conditions is generated. func SetAggregate(to Setter, targetCondition string, from []Getter, options ...MergeOption) { diff --git a/pkg/conditions2/setter_test.go b/pkg/conditions2/setter_test.go index ceb78d52a..5d9e47b25 100644 --- a/pkg/conditions2/setter_test.go +++ b/pkg/conditions2/setter_test.go @@ -199,7 +199,7 @@ func TestMarkMethods(t *testing.T) { g.Expect(Get(vm, "conditionFoo")).To(haveSameStateOf(&metav1.Condition{ Type: "conditionFoo", Status: metav1.ConditionTrue, - Reason: "conditionFoo", + Reason: "True", })) // test MarkFalse diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index c1d05aca6..374763b4a 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -29,6 +29,7 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/record" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere" + vsphere2 "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" ) // Manager is a VM Operator controller manager. @@ -116,7 +117,12 @@ func New(opts Options) (Manager, error) { func InitializeProviders(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { vmProviderName := fmt.Sprintf("%s/%s/vmProvider", ctx.Namespace, ctx.Name) recorder := record.New(mgr.GetEventRecorderFor(vmProviderName)) - ctx.VMProvider = vsphere.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + + if lib.IsVMServiceV1Alpha2FSSEnabled() { + ctx.VMProviderA2 = vsphere2.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + } else { + ctx.VMProvider = vsphere.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + } return nil } diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go index 899ff5015..0c3d3aa2b 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go @@ -146,6 +146,7 @@ func vmTests() { ) BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed testConfig.WithVMClassAsConfigDaynDate = true ethCard = types.VirtualEthernetCard{ @@ -1590,6 +1591,8 @@ func vmTests() { Context("Multiple NICs are specified", func() { BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + vm.Spec.NetworkInterfaces = []vmopv1.VirtualMachineNetworkInterface{ { NetworkName: "VM Network", diff --git a/pkg/vmprovider/providers/vsphere2/client/client.go b/pkg/vmprovider/providers/vsphere2/client/client.go new file mode 100644 index 000000000..7477fd621 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client.go @@ -0,0 +1,255 @@ +// Copyright (c) 2018-2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "context" + "net" + "net/url" + "time" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/session/keepalive" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/methods" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +var log = logf.Log.WithName("vsphere").WithName("client") + +type Client struct { + vimClient *vim25.Client + finder *find.Finder + datacenter *object.Datacenter + restClient *rest.Client + contentLibClient contentlibrary.Provider + clusterModClient clustermodules.Provider + sessionManager *session.Manager + config *config.VSphereVMProviderConfig +} + +// Idle time before a keepalive will be invoked. +const keepAliveIdleTime = 5 * time.Minute + +// SoapKeepAliveHandlerFn returns a keepalive handler function suitable for use with the SOAP handler. +// In case the connectivity to VC is down long enough, the session expires. Further attempts to use the +// client yield NotAuthenticated fault. This handler ensures that we re-login the client in those scenarios. +func SoapKeepAliveHandlerFn(sc *soap.Client, sm *session.Manager, userInfo *url.Userinfo) func() error { + return func() error { + ctx := context.Background() + if _, err := methods.GetCurrentTime(ctx, sc); err != nil && isNotAuthenticatedError(err) { + log.Info("Re-authenticating vim client") + if err = sm.Login(ctx, userInfo); err != nil { + if isInvalidLogin(err) { + log.Error(err, "Invalid login in keepalive handler", "url", sc.URL()) + return err + } + } + } else if err != nil { + log.Error(err, "Error in vim25 client's keepalive handler", "url", sc.URL()) + } + + return nil + } +} + +// RestKeepAliveHandlerFn returns a keepalive handler function suitable for use with the REST handler. +// Similar to the SOAP handler, we customize the handler here so we can re-login the client in case the +// REST session expires due to connectivity issues. +func RestKeepAliveHandlerFn(c *rest.Client, userInfo *url.Userinfo) func() error { + return func() error { + ctx := context.Background() + if sess, err := c.Session(ctx); err == nil && sess == nil { + // session is Unauthorized. + log.Info("Re-authenticating REST client") + if err = c.Login(ctx, userInfo); err != nil { + log.Error(err, "Invalid login in keepalive handler", "url", c.URL()) + return err + } + } else if err != nil { + log.Error(err, "Error in rest client's keepalive handler", "url", c.URL()) + } + + return nil + } +} + +// newRestClient creates a rest client which is configured to use a custom keepalive handler function. +func newRestClient(ctx context.Context, vimClient *vim25.Client, config *config.VSphereVMProviderConfig) (*rest.Client, error) { + log.Info("Creating new REST Client", "VcPNID", config.VcPNID, "VcPort", config.VcPort) + restClient := rest.NewClient(vimClient) + + userInfo := url.UserPassword(config.VcCreds.Username, config.VcCreds.Password) + + // Set a custom keepalive handler function + restClient.Transport = keepalive.NewHandlerREST(restClient, keepAliveIdleTime, RestKeepAliveHandlerFn(restClient, userInfo)) + + // Initial login. This will also start the keepalive. + if err := restClient.Login(ctx, userInfo); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "login failed for url: %v", vimClient.URL()) + } + + return restClient, nil +} + +// NewVimClient creates a new vim25 client which is configured to use a custom keepalive handler function. +// Making this public to allow access from other packages when only VimClient is needed. +func NewVimClient(ctx context.Context, config *config.VSphereVMProviderConfig) (*vim25.Client, *session.Manager, error) { + log.Info("Creating new vim Client", "VcPNID", config.VcPNID, "VcPort", config.VcPort) + soapURL, err := soap.ParseURL(net.JoinHostPort(config.VcPNID, config.VcPort)) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse %s:%s", config.VcPNID, config.VcPort) + } + + soapClient := soap.NewClient(soapURL, config.InsecureSkipTLSVerify) + if config.CAFilePath != "" { + err = soapClient.SetRootCAs(config.CAFilePath) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to set root CA %s", config.CAFilePath) + } + } + + vimClient, err := vim25.NewClient(ctx, soapClient) + if err != nil { + return nil, nil, errors.Wrapf(err, "error creating a new vim client for url: %v", soapURL) + } + + if err := vimClient.UseServiceVersion(); err != nil { + return nil, nil, errors.Wrapf(err, "error setting vim client version for url: %v", soapURL) + } + + userInfo := url.UserPassword(config.VcCreds.Username, config.VcCreds.Password) + sm := session.NewManager(vimClient) + + // Set a custom keepalive handler function + vimClient.RoundTripper = keepalive.NewHandlerSOAP(soapClient, keepAliveIdleTime, SoapKeepAliveHandlerFn(soapClient, sm, userInfo)) + + // Initial login. This will also start the keepalive. + if err = sm.Login(ctx, userInfo); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, nil, errors.Wrapf(err, "login failed for url: %v", soapURL) + } + + return vimClient, sm, err +} + +func newFinder( + ctx context.Context, + vimClient *vim25.Client, + config *config.VSphereVMProviderConfig) (*find.Finder, *object.Datacenter, error) { + + finder := find.NewFinder(vimClient, false) + + dcRef, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "Datacenter", Value: config.Datacenter}) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to find Datacenter %q", config.Datacenter) + } + + dc := dcRef.(*object.Datacenter) + finder.SetDatacenter(dc) + + return finder, dc, nil +} + +// NewClient creates a new Client. As a side effect, it creates a vim25 client and a REST client. +func NewClient(ctx context.Context, config *config.VSphereVMProviderConfig) (*Client, error) { + vimClient, sm, err := NewVimClient(ctx, config) + if err != nil { + return nil, err + } + + finder, datacenter, err := newFinder(ctx, vimClient, config) + if err != nil { + return nil, err + } + + restClient, err := newRestClient(ctx, vimClient, config) + if err != nil { + return nil, err + } + + return &Client{ + vimClient: vimClient, + finder: finder, + datacenter: datacenter, + restClient: restClient, + contentLibClient: contentlibrary.NewProvider(restClient), + clusterModClient: clustermodules.NewProvider(restClient), + sessionManager: sm, + config: config, + }, nil +} + +func isNotAuthenticatedError(err error) bool { + if soap.IsSoapFault(err) { + vimFault := soap.ToSoapFault(err).VimFault() + if _, ok := vimFault.(types.NotAuthenticated); ok { + return true + } + } + + return false +} + +func isInvalidLogin(err error) bool { + if soap.IsSoapFault(err) { + vimFault := soap.ToSoapFault(err).VimFault() + if _, ok := vimFault.(types.InvalidLogin); ok { + return true + } + } + + return false +} + +func (c *Client) VimClient() *vim25.Client { + return c.vimClient +} + +func (c *Client) Finder() *find.Finder { + return c.finder +} + +func (c *Client) Datacenter() *object.Datacenter { + return c.datacenter +} + +func (c *Client) RestClient() *rest.Client { + return c.restClient +} + +func (c *Client) ContentLibClient() contentlibrary.Provider { + return c.contentLibClient +} + +func (c *Client) ClusterModuleClient() clustermodules.Provider { + return c.clusterModClient +} + +func (c *Client) Config() *config.VSphereVMProviderConfig { + return c.config +} + +func (c *Client) Logout(ctx context.Context) { + clientURL := c.vimClient.URL() + log.Info("vsphere client logging out from", "VC", clientURL.Host) + if err := c.sessionManager.Logout(ctx); err != nil { + log.Error(err, "Error logging out the vim25 session", "username", clientURL.User.Username(), "host", clientURL.Host) + } + + if err := c.restClient.Logout(ctx); err != nil { + log.Error(err, "Error logging out the rest session", "username", clientURL.User.Username(), "host", clientURL.Host) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go b/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go new file mode 100644 index 000000000..d77790a4f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go @@ -0,0 +1,46 @@ +//go:build !race + +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +var ( + model *simulator.Model + server *simulator.Server + ctx context.Context + tlsTestModel *simulator.Model + tlsServer *simulator.Server + tlsServerCertPath string + tlsServerKeyPath string +) + +var _ = BeforeSuite(func() { + ctx, model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer = test.BeforeSuite() +}) + +var _ = AfterSuite(func() { + test.AfterSuite( + ctx, + model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer) +}) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Client Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/client/client_test.go b/pkg/vmprovider/providers/vsphere2/client/client_test.go new file mode 100644 index 000000000..a5e3aec21 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client_test.go @@ -0,0 +1,366 @@ +//go:build !race + +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client_test + +import ( + "context" + "net/url" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/session/keepalive" + "github.com/vmware/govmomi/simulator" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + + . "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +func testConfig(vcpnid string, vcport string, user string, pass string) *config.VSphereVMProviderConfig { + providerConfig := &config.VSphereVMProviderConfig{ + VcPNID: vcpnid, + VcPort: vcport, + Datacenter: "datacenter-2", + VcCreds: &credentials.VSphereVMProviderCredentials{ + Username: user, + Password: pass, + }, + // Let the tests run without TLS validation by default. + InsecureSkipTLSVerify: true, + } + return providerConfig +} + +var _ = Describe("keepalive handler", func() { + + var ( + sessionIdleTimeout = time.Second / 2 + keepAliveIdle = sessionIdleTimeout / 2 + sessionCheckPause = 3 * sessionIdleTimeout // Avoid unit test thread waking up before session expires + simulatorIdleTime time.Duration + ) + + assertSoapSessionValid := func(ctx context.Context, m *session.Manager) { + s, err := m.UserSession(ctx) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, s).NotTo(BeNil()) + } + + assertSoapSessionExpired := func(ctx context.Context, m *session.Manager) { + s, err := m.UserSession(ctx) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, s).To(BeNil()) + } + + BeforeEach(func() { + // Sharing this variable causes the race detector to trip. + simulator.SessionIdleTimeout = sessionIdleTimeout + }) + + AfterEach(func() { + simulator.SessionIdleTimeout = simulatorIdleTime + }) + + Context("for SOAP sessions", func() { + Context("with a logged out session)", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + m := session.NewManager(c) + + // Session should be valid since simulator logs in the client by default + assertSoapSessionValid(ctx, m) + + // Sleep for time > sessionIdleTimeout, so the session expires + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m) + + // Set the keepalive handler + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m, simulator.DefaultLogin)) + + // Start the handler + Expect(m.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + time.Sleep(sessionCheckPause) + + // Session should not have been expired + assertSoapSessionValid(ctx, m) + + Expect(m.Logout(ctx)).To(Succeed()) + assertSoapSessionExpired(ctx, m) + }) + }) + }) + + getNewSessionManager := func(url *url.URL) *session.Manager { + c2, err := vim25.NewClient(ctx, soap.NewClient(url, true)) + Expect(err).NotTo(HaveOccurred()) + Expect(c2).NotTo(BeNil()) + + // With default keepalive handler + m2 := session.NewManager(c2) + + c2.RoundTripper = keepalive.NewHandlerSOAP(c2.RoundTripper, keepAliveIdle, nil) + + // Start the handler + Expect(m2.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + return m2 + } + + // To test NotAuthenticated Fault: + // We cannot "self-terminate" a session. So, this following two tests creates _two_ session managers. One with + // the custom keepalive handler, and another session to orchestrate the "NotAuthenticated" fault. + // We terminate the first session using the second session's manager. + + Context("Re-login on NotAuthenticated: with a session that is terminated", func() { + Context("when handler is called with correct userInfo", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + + // With custom keepalive handler + m1 := session.NewManager(c) + // Orchestrator session + m2 := getNewSessionManager(c.URL()) + + // Session should be valid + assertSoapSessionValid(ctx, m1) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m1) + + // set the keepalive handler + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m1, simulator.DefaultLogin)) + + // Start the handler + Expect(m1.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Wait for the keepalive handler to get called + time.Sleep(sessionCheckPause) + + // Session should be valid since Login starts a new one + assertSoapSessionValid(ctx, m1) + + // Terminate session to emulate NotAuthenticated Error + sess, err := m1.UserSession(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(sess).NotTo(BeNil()) + + By("terminating the session") + Expect(m2.TerminateSession(ctx, []string{sess.Key})).To(Succeed()) + + // session expired since it is terminated + assertSoapSessionExpired(ctx, m1) + + time.Sleep(sessionCheckPause) + + // keepalive handler must have re-logged in + assertSoapSessionValid(ctx, m1) + }) + }) + }) + }) + + Context("Relogin on NotAuthenticated: with a session that is terminated", func() { + Context("when handler is called with wrong userInfo", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + + // With custom keepalive handler + m1 := session.NewManager(c) + // With default keepalive handler + m2 := getNewSessionManager(c.URL()) + + // Session should be valid + assertSoapSessionValid(ctx, m1) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m1) + + // set the keepalive handler with wrong userInfo + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m1, nil)) + + // Start the handler + Expect(m1.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Wait for the keepalive handler to get called + time.Sleep(sessionCheckPause) + + // Session should be valid since Login starts a new one + assertSoapSessionValid(ctx, m1) + + // Terminate session to emulate NotAuthenticated Error + sess, err := m1.UserSession(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(sess).NotTo(BeNil()) + + By("terminating the session") + Expect(m2.TerminateSession(ctx, []string{sess.Key})).To(Succeed()) + + // session expired since it is terminated + assertSoapSessionExpired(ctx, m1) + + // Wait for keepalive handler to be called + time.Sleep(sessionCheckPause) + + // keepalive handler should error out, session still invalid + assertSoapSessionExpired(ctx, m1) + }) + }) + }) + }) + }) + + assertRestSessionValid := func(ctx context.Context, c *rest.Client) { + s, err := c.Session(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(s).NotTo(BeNil()) + } + + assertRestSessionExpired := func(ctx context.Context, c *rest.Client) { + s, err := c.Session(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(s).To(BeNil()) + } + + Context("REST sessions", func() { + Context("When a REST session is logged out)", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, vc *vim25.Client) { + c := rest.NewClient(vc) + + Expect(c.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Session should be valid + assertRestSessionValid(ctx, c) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertRestSessionExpired(ctx, c) + + // Set the keepalive handler + c.Transport = keepalive.NewHandlerREST(c, keepAliveIdle, RestKeepAliveHandlerFn(c, simulator.DefaultLogin)) + + // Start the handler + Expect(c.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Session should not have been expired + assertRestSessionValid(ctx, c) + + Expect(c.Logout(ctx)).To(Succeed()) + assertRestSessionExpired(ctx, c) + }) + }) + }) + }) +}) + +var _ = Describe("NewClient", func() { + Context("When called with valid config", func() { + Specify("returns a valid client and no error", func() { + client, err := NewClient(ctx, testConfig(server.URL.Hostname(), server.URL.Port(), "some-username", "some-password")) + Expect(err).ToNot(HaveOccurred()) + Expect(client).ToNot(BeNil()) + }) + }) + + Context("When called with invalid host and port", func() { + Specify("soap.ParseURL should fail", func() { + failConfig := testConfig("test%test", "", "test-user", "test-pass") + client, err := NewClient(ctx, failConfig) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to parse")) + Expect(client).To(BeNil()) + }) + }) + + Context("When called with invalid VC PNID", func() { + Specify("returns failed to parse error", func() { + failConfig := testConfig("test-pnid", "test-port", "test-user", "test-pass") + client, err := NewClient(ctx, failConfig) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid")) + Expect(err.Error()).To(ContainSubstring("port")) + Expect(err.Error()).To(ContainSubstring("test-port")) + Expect(client).To(BeNil()) + }) + }) + + DescribeTable("Should fail if given wrong username and/or wrong password", + func(expectedUsername, expectedPassword, username, password string) { + server.URL.User = url.UserPassword(expectedUsername, expectedPassword) + model.Service.Listen = server.URL + config := testConfig(server.URL.Hostname(), server.URL.Port(), username, password) + client, err := NewClient(ctx, config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("login failed for url")) + Expect(client).To(BeNil()) + }, + Entry("with wrong username and password", "correct-username", "correct-password", "username", "password"), + Entry("with wrong username and correct password", "correct-username", "correct-password", "username", "correct-password"), + Entry("with correct username and wrong password", "correct-username", "correct-password", "correct-username", "password"), + ) +}) + +// Most of the other VM Operator tests run without TLS verification. Start up a separate simulator with a fresh TLS key/cert +// +// and ensure the client can connect to it. +var _ = Describe("Tests for client TLS", func() { + Context("when the client recognizes the certificate presented by the VC", func() { + It("successfully connects to the VC", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + config.InsecureSkipTLSVerify = false + config.CAFilePath = tlsServerCertPath + _, err := NewClient(context.Background(), config) + + Expect(err).NotTo(HaveOccurred()) + }) + }) + Context("when the CA bundle referred to does not exist", func() { + It("returns an error about loading the CA bundle", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + config.CAFilePath = "/a/nonexistent/ca-bundle.crt" + config.InsecureSkipTLSVerify = false + _, err := NewClient(context.Background(), config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to set root CA")) + }) + + }) + Context("when the client does not recognize the certificate presented by the VC", func() { + Context("when TLS verification is used", func() { + It("returns an error about certificate validation", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + _, randomCANotForServer := test.GenerateSelfSignedCert() + config.CAFilePath = randomCANotForServer + config.InsecureSkipTLSVerify = false + _, err := NewClient(context.Background(), config) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("x509: certificate signed by unknown authority")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go new file mode 100644 index 000000000..7e8208443 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go @@ -0,0 +1,8 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import logf "sigs.k8s.io/controller-runtime/pkg/log" + +var log = logf.Log.WithName("vsphere").WithName("clustermodules") diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go new file mode 100644 index 000000000..ce21d3dd6 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go @@ -0,0 +1,127 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import ( + "context" + + "github.com/vmware/govmomi/vapi/cluster" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type Provider interface { + CreateModule(ctx context.Context, clusterRef types.ManagedObjectReference) (string, error) + DeleteModule(ctx context.Context, moduleID string) error + DoesModuleExist(ctx context.Context, moduleID string, cluster types.ManagedObjectReference) (bool, error) + + IsMoRefModuleMember(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) (bool, error) + AddMoRefToModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error + RemoveMoRefFromModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error +} + +type provider struct { + manager *cluster.Manager +} + +func NewProvider(restClient *rest.Client) Provider { + return &provider{ + manager: cluster.NewManager(restClient), + } +} + +func (cm *provider) CreateModule(ctx context.Context, clusterRef types.ManagedObjectReference) (string, error) { + log.Info("Creating cluster module", "cluster", clusterRef) + + moduleID, err := cm.manager.CreateModule(ctx, clusterRef) + if err != nil { + return "", err + } + + log.Info("Created cluster module", "moduleID", moduleID) + return moduleID, nil +} + +func (cm *provider) DeleteModule(ctx context.Context, moduleID string) error { + log.Info("Deleting cluster module", "moduleID", moduleID) + + err := cm.manager.DeleteModule(ctx, moduleID) + if err != nil && !lib.IsNotFoundError(err) { + return err + } + + log.Info("Deleted cluster module", "moduleID", moduleID) + return nil +} + +func (cm *provider) DoesModuleExist(ctx context.Context, moduleID string, clusterRef types.ManagedObjectReference) (bool, error) { + log.V(4).Info("Checking if cluster module exists", "moduleID", moduleID, "clusterRef", clusterRef) + + if moduleID == "" { + return false, nil + } + + // This is not efficient for as we use DoesModuleExist(). + modules, err := cm.manager.ListModules(ctx) + if err != nil { + return false, err + } + + for _, mod := range modules { + if mod.Cluster == clusterRef.Value && mod.Module == moduleID { + return true, nil + } + } + + log.V(4).Info("Cluster module doesn't exist", "moduleID", moduleID, "clusterRef", clusterRef) + return false, nil +} + +func (cm *provider) IsMoRefModuleMember(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) (bool, error) { + moduleMembers, err := cm.manager.ListModuleMembers(ctx, moduleID) + if err != nil { + return false, err + } + + for _, member := range moduleMembers { + if member.Reference() == moRef.Reference() { + return true, nil + } + } + + return false, nil +} + +func (cm *provider) AddMoRefToModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error { + isMember, err := cm.IsMoRefModuleMember(ctx, moduleID, moRef) + if err != nil { + return err + } + + if !isMember { + log.Info("Adding moRef to cluster module", "moduleID", moduleID, "moRef", moRef) + // TODO: Should we just skip the IsMoRefModuleMember() and always call this since we're already + // ignoring the first return value? + _, err := cm.manager.AddModuleMembers(ctx, moduleID, moRef.Reference()) + if err != nil { + return err + } + } + + return nil +} + +func (cm *provider) RemoveMoRefFromModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error { + log.Info("Removing moRef from cluster module", "moduleID", moduleID, "moRef", moRef) + + _, err := cm.manager.RemoveModuleMembers(ctx, moduleID, moRef) + if err != nil { + return err + } + + log.Info("Removed moRef from cluster module", "moduleID", moduleID, "moRef", moRef) + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go new file mode 100644 index 000000000..e46a72346 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ClusterModules Provider", cmTests) +} + +var suite = builder.NewTestSuite() + +func TestClusterModules(t *testing.T) { + suite.Register(t, "vSphere Provider Cluster Modules Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go new file mode 100644 index 000000000..67ce19011 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func cmTests() { + Describe("Cluster Modules", func() { + + var ( + ctx *builder.TestContextForVCSim + cmProvider clustermodules.Provider + + moduleGroup string + moduleSpec *vmopv1.ClusterModuleSpec + moduleStatus *vmopv1.ClusterModuleStatus + clusterRef types.ManagedObjectReference + vmRef types.ManagedObjectReference + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + cmProvider = clustermodules.NewProvider(ctx.RestClient) + + clusterRef = ctx.GetSingleClusterCompute().Reference() + + moduleGroup = "controller-group" + moduleSpec = &vmopv1.ClusterModuleSpec{ + GroupName: moduleGroup, + } + + moduleID, err := cmProvider.CreateModule(ctx, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(moduleID).ToNot(BeEmpty()) + + moduleStatus = &vmopv1.ClusterModuleStatus{ + GroupName: moduleSpec.GroupName, + ModuleUuid: moduleID, + } + + // TODO: Create VM instead of using one that vcsim creates for free. + vm, err := ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + vmRef = vm.Reference() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Create a ClusterModule, verify it exists and delete it", func() { + exists, err := cmProvider.DoesModuleExist(ctx, moduleStatus.ModuleUuid, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + + Expect(cmProvider.DeleteModule(ctx, moduleStatus.ModuleUuid)).To(Succeed()) + + exists, err = cmProvider.DoesModuleExist(ctx, moduleStatus.ModuleUuid, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + Context("ClusterModule-VM association", func() { + It("check membership doesn't exist", func() { + isMember, err := cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeFalse()) + }) + + It("Associate a VM with a clusterModule, check the membership and remove it", func() { + By("Associate VM") + err := cmProvider.AddMoRefToModule(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + + By("Verify membership") + isMember, err := cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeTrue()) + + By("Remove the association") + err = cmProvider.RemoveMoRefFromModule(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + + By("Verify no longer a member") + isMember, err = cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeFalse()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go new file mode 100644 index 000000000..898a7d82e --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go @@ -0,0 +1,62 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import ( + "context" + + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +// FindClusterModuleUUID returns the index in the Status.ClusterModules and UUID of the +// VC cluster module for the given groupName and cluster reference. +func FindClusterModuleUUID( + groupName string, + clusterRef types.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (int, string) { + + // Prior to the stretched cluster work, the status did not contain the VC cluster the module was + // created for, but we still need to return existing modules when the FSS is not enabled. + matchCluster := lib.IsWcpFaultDomainsFSSEnabled() + + for i, modStatus := range resourcePolicy.Status.ClusterModules { + if modStatus.GroupName == groupName && (modStatus.ClusterMoID == clusterRef.Value || !matchCluster) { + return i, modStatus.ModuleUuid + } + } + + return -1, "" +} + +// ClaimClusterModuleUUID tries to find an existing entry in the Status.ClusterModules that is for +// the given groupName and cluster reference. This is meant for after an upgrade where the FaultDomains +// FSS is now enabled but we had not previously set the ClusterMoID. +func ClaimClusterModuleUUID( + ctx context.Context, + clusterModProvider Provider, + groupName string, + clusterRef types.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (int, string, error) { + + var errs []error + + if lib.IsWcpFaultDomainsFSSEnabled() { + for i, modStatus := range resourcePolicy.Status.ClusterModules { + if modStatus.GroupName == groupName && modStatus.ClusterMoID == "" { + exists, err := clusterModProvider.DoesModuleExist(ctx, modStatus.ModuleUuid, clusterRef) + if err != nil { + errs = append(errs, err) + } else if exists { + return i, modStatus.ModuleUuid, nil + } + } + } + } + + return -1, "", k8serrors.NewAggregate(errs) +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go new file mode 100644 index 000000000..db3e83241 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" +) + +var _ = Describe("FindClusterModuleUUID", func() { + const ( + groupName1, groupName2 = "groupName1", "groupName2" + moduleUUID1, moduleUUID2 = "uuid1", "uuid2" + ) + + var ( + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + clusterRef1, clusterRef2 types.ManagedObjectReference + ) + + BeforeEach(func() { + resourcePolicy = &vmopv1.VirtualMachineSetResourcePolicy{ + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ClusterModuleGroups: []string{groupName1, groupName2}, + }, + } + + clusterRef1 = types.ManagedObjectReference{Value: "dummy-cluster1"} + clusterRef2 = types.ManagedObjectReference{Value: "dummy-cluster2"} + }) + + Context("FaultDomains FSS is disabled", func() { + + Context("GroupName does not exist", func() { + It("Returns expected values", func() { + idx, uuid := clustermodules.FindClusterModuleUUID("does-not-exist", clusterRef1, resourcePolicy) + Expect(idx).To(Equal(-1)) + Expect(uuid).To(BeEmpty()) + }) + }) + + Context("GroupName exists", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName2, + ModuleUuid: moduleUUID2, + ClusterMoID: "this should be ignored", + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName2, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + + }) + + Context("FaultDomains FSS is enabled", func() { + var ( + oldFaultDomainsFunc func() bool + ) + + BeforeEach(func() { + oldFaultDomainsFunc = lib.IsWcpFaultDomainsFSSEnabled + lib.IsWcpFaultDomainsFSSEnabled = func() bool { return true } + }) + + AfterEach(func() { + lib.IsWcpFaultDomainsFSSEnabled = oldFaultDomainsFunc + }) + + Context("GroupName does not exist", func() { + It("Returns expected values", func() { + idx, uuid := clustermodules.FindClusterModuleUUID("does-not-exist", clusterRef1, resourcePolicy) + Expect(idx).To(Equal(-1)) + Expect(uuid).To(BeEmpty()) + }) + }) + + Context("GroupName exists", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + ClusterMoID: clusterRef1.Value, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName2, + ModuleUuid: moduleUUID2, + ClusterMoID: clusterRef1.Value, + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName2, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + + Context("Matches by cluster reference", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + ClusterMoID: clusterRef1.Value, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID2, + ClusterMoID: clusterRef2.Value, + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName1, clusterRef2, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/config/config.go b/pkg/vmprovider/providers/vsphere2/config/config.go new file mode 100644 index 000000000..8d2071a42 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config.go @@ -0,0 +1,290 @@ +// Copyright (c) 2018-2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "context" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/pkg/errors" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" +) + +var log = logf.Log.WithName("vsphere").WithName("config") + +// VSphereVMProviderConfig represents the configuration for a Vsphere VM Provider instance. +// Contains information enabling integration with a backend vSphere instance for VM management. +type VSphereVMProviderConfig struct { + VcPNID string + VcPort string + VcCreds *credentials.VSphereVMProviderCredentials + Datacenter string + StorageClassRequired bool // Always true in WCP env. + UseInventoryAsContentSource bool // Always false in WCP env. + CAFilePath string + InsecureSkipTLSVerify bool // Always false in WCP env. + + // These are Zone and/or Namespace specific. + ResourcePool string + Folder string + + // Only set in simulated testing env. + Datastore string + Network string +} + +const ( + DefaultVCPort = "443" + + ProviderConfigMapName = "vsphere.provider.config.vmoperator.vmware.com" + // Keys in provider ConfigMap. + vcPNIDKey = "VcPNID" + vcPortKey = "VcPort" + vcCredsSecretNameKey = "VcCredsSecretName" //nolint:gosec + datacenterKey = "Datacenter" + resourcePoolKey = "ResourcePool" + folderKey = "Folder" + datastoreKey = "Datastore" + networkNameKey = "Network" + scRequiredKey = "StorageClassRequired" + useInventoryKey = "UseInventoryAsContentSource" + insecureSkipTLSVerifyKey = "InsecureSkipTLSVerify" + caFilePathKey = "CAFilePath" + ContentSourceKey = "ContentSource" + + NetworkConfigMapName = "vmoperator-network-config" + NameserversKey = "nameservers" // Key in the NetworkConfigMapName. + SearchSuffixesKey = "searchsuffixes" // Key in the NetworkConfigMapName. +) + +// ConfigMapToProviderConfig converts the VM provider ConfigMap to a VSphereVMProviderConfig. +func ConfigMapToProviderConfig( //nolint: revive // Ignore linter error about stuttering. + configMap *corev1.ConfigMap, + vcCreds *credentials.VSphereVMProviderCredentials) (*VSphereVMProviderConfig, error) { + + vcPNID, ok := configMap.Data[vcPNIDKey] + if !ok { + return nil, errors.New("missing configMap data field VcPNID") + } + + vcPort, ok := configMap.Data[vcPortKey] + if !ok { + vcPort = DefaultVCPort + } + + scRequired := false + if s, ok := configMap.Data[scRequiredKey]; ok { + var err error + scRequired, err = strconv.ParseBool(s) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of StorageClassRequired") + } + } + + useInventory := false + if u, ok := configMap.Data[useInventoryKey]; ok { + var err error + useInventory, err = strconv.ParseBool(u) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of UseInventory") + } + } + + // Default to validating TLS. + insecureSkipTLSVerify := false + if v, ok := configMap.Data[insecureSkipTLSVerifyKey]; ok { + var err error + insecureSkipTLSVerify, err = strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of InsecureSkipTLSVerify") + } + } + + var caFilePath string + if ca, ok := configMap.Data[caFilePathKey]; !insecureSkipTLSVerify && ok { + // The value will be /etc/vmware/wcp/tls/vmca.pem. While this is from our provider ConfigMap + // it must match the volume path in our Deployment. + caFilePath = ca + } + + ret := &VSphereVMProviderConfig{ + VcPNID: vcPNID, + VcPort: vcPort, + VcCreds: vcCreds, + Datacenter: configMap.Data[datacenterKey], + ResourcePool: configMap.Data[resourcePoolKey], + Folder: configMap.Data[folderKey], + Datastore: configMap.Data[datastoreKey], + Network: configMap.Data[networkNameKey], + StorageClassRequired: scRequired, + UseInventoryAsContentSource: useInventory, + InsecureSkipTLSVerify: insecureSkipTLSVerify, + CAFilePath: caFilePath, + } + + return ret, nil +} + +func configMapToProviderCredentials( + client ctrlruntime.Client, + configMap *corev1.ConfigMap) (*credentials.VSphereVMProviderCredentials, error) { + + secretName := configMap.Data[vcCredsSecretNameKey] + if secretName == "" { + return nil, errors.Errorf("%s creds secret not set in vmop system namespace", vcCredsSecretNameKey) + } + + return credentials.GetProviderCredentials(client, configMap.Namespace, secretName) +} + +func GetDNSInformationFromConfigMap(client ctrlruntime.Client) ([]string, []string, error) { + vmopNamespace, err := lib.GetVMOpNamespaceFromEnv() + if err != nil { + return nil, nil, err + } + + configMap := &corev1.ConfigMap{} + configMapKey := ctrlruntime.ObjectKey{Name: NetworkConfigMapName, Namespace: vmopNamespace} + if err := client.Get(context.Background(), configMapKey, configMap); err != nil { + return nil, nil, err + } + + var ( + nameservers []string + searchSuffixes []string + ) + + nsStr, ok := configMap.Data[NameserversKey] + if !ok { + return nil, nil, errors.Wrapf(err, "invalid %v ConfigMap, missing key nameservers", NetworkConfigMapName) + } + + nameservers = strings.Fields(nsStr) + if len(nameservers) == 0 { + return nil, nil, errors.Errorf("No nameservers in %v ConfigMap", NetworkConfigMapName) + } + + if len(nameservers) == 1 && nameservers[0] == "" { + return nil, nil, errors.Errorf("No valid nameservers in %v ConfigMap. It still contains key", NetworkConfigMapName) + } + + if ssStr, ok := configMap.Data[SearchSuffixesKey]; ok { + searchSuffixes = strings.Fields(ssStr) + } + + // do we need to validate that these look like valid ipv4 addresses? + return nameservers, searchSuffixes, nil +} + +// getProviderConfigMap returns the provider ConfigMap. +func getProviderConfigMap( + ctx context.Context, + client ctrlruntime.Client) (*corev1.ConfigMap, error) { + + vmopNamespace, err := lib.GetVMOpNamespaceFromEnv() + if err != nil { + return nil, err + } + + configMap := &corev1.ConfigMap{} + configMapKey := ctrlruntime.ObjectKey{Name: ProviderConfigMapName, Namespace: vmopNamespace} + if err := client.Get(ctx, configMapKey, configMap); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "error retrieving the provider ConfigMap %s", configMapKey) + } + + return configMap, nil +} + +// GetProviderConfig returns a provider config constructed from vSphere Provider ConfigMap in the VM Operator namespace. +func GetProviderConfig( + ctx context.Context, + client ctrlruntime.Client) (*VSphereVMProviderConfig, error) { + + configMap, err := getProviderConfigMap(ctx, client) + if err != nil { + return nil, err + } + + vcCreds, err := configMapToProviderCredentials(client, configMap) + if err != nil { + return nil, err + } + + providerConfig, err := ConfigMapToProviderConfig(configMap, vcCreds) + if err != nil { + return nil, err + } + + return providerConfig, nil +} + +func setConfigMapData(configMap *corev1.ConfigMap, config *VSphereVMProviderConfig, vcCredsSecretName string) { + if configMap.Data == nil { + configMap.Data = map[string]string{} + } + + configMap.Data[vcPNIDKey] = config.VcPNID + configMap.Data[vcPortKey] = config.VcPort + configMap.Data[vcCredsSecretNameKey] = vcCredsSecretName + configMap.Data[datacenterKey] = config.Datacenter + configMap.Data[resourcePoolKey] = config.ResourcePool + configMap.Data[folderKey] = config.Folder + configMap.Data[datastoreKey] = config.Datastore + configMap.Data[scRequiredKey] = strconv.FormatBool(config.StorageClassRequired) + configMap.Data[useInventoryKey] = strconv.FormatBool(config.UseInventoryAsContentSource) + configMap.Data[caFilePathKey] = config.CAFilePath + configMap.Data[insecureSkipTLSVerifyKey] = strconv.FormatBool(config.InsecureSkipTLSVerify) +} + +// ProviderConfigToConfigMap returns the ConfigMap for the config. +// Used only in testing. +func ProviderConfigToConfigMap( + namespace string, + config *VSphereVMProviderConfig, + vcCredsSecretName string) *corev1.ConfigMap { + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProviderConfigMapName, + Namespace: namespace, + }, + } + setConfigMapData(configMap, config, vcCredsSecretName) + + return configMap +} + +// UpdateVcInConfigMap updates the ConfigMap with the new vCenter PNID and Port. Returns false if no updated needed. +func UpdateVcInConfigMap(ctx context.Context, client ctrlruntime.Client, vcPNID, vcPort string) (bool, error) { + configMap, err := getProviderConfigMap(ctx, client) + if err != nil { + return false, err + } + + if configMap.Data[vcPNIDKey] == vcPNID && configMap.Data[vcPortKey] == vcPort { + // No update needed. + return false, nil + } + + origConfigMap := configMap.DeepCopy() + configMap.Data[vcPNIDKey] = vcPNID + configMap.Data[vcPortKey] = vcPort + + err = client.Patch(ctx, configMap, ctrlruntime.MergeFrom(origConfigMap)) + if err != nil { + log.Error(err, "Failed to update provider ConfigMap", "configMapName", configMap.Name) + return false, err + } + + return true, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go b/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go new file mode 100644 index 000000000..c82c0e933 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("Config", configTests) +} + +var suite = builder.NewTestSuite() + +func TestConfig(t *testing.T) { + suite.Register(t, "vSphere Provider Config Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/config/config_test.go b/pkg/vmprovider/providers/vsphere2/config/config_test.go new file mode 100644 index 000000000..28bda78bb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func configTests() { + + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("GetProviderConfig", func() { + + Context("GetProviderConfig", func() { + + Context("when a secret doesn't exist", func() { + It("returns no provider config and an error", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmop-vcsim-dummy-creds", + Namespace: ctx.PodNamespace, + }, + } + Expect(ctx.Client.Delete(ctx, secret)).To(Succeed()) + + providerConfig, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).To(HaveOccurred()) + Expect(providerConfig).To(BeNil()) + }) + }) + + Context("when a secret exists", func() { + It("returns a good provider config", func() { + _, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) + }) + + Describe("UpdateVcInConfigMap", func() { + + Context("UpdateVcInConfigMap", func() { + DescribeTable("Update VC PNID and VC Port", + func(newPnid string, newPort string) { + expectedUpdated := newPnid != "" || newPort != "" + + providerConfig, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).ToNot(HaveOccurred()) + if newPnid == "" { + newPnid = providerConfig.VcPNID + } + if newPort == "" { + newPort = providerConfig.VcPort + } + + updated, err := config.UpdateVcInConfigMap(ctx, ctx.Client, newPnid, newPort) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(Equal(expectedUpdated)) + + providerConfig, err = config.GetProviderConfig(ctx, ctx.Client) + Expect(err).NotTo(HaveOccurred()) + Expect(providerConfig.VcPNID).To(Equal(newPnid)) + Expect(providerConfig.VcPort).To(Equal(newPort)) + }, + Entry("only VC PNID is updated", "some-pnid", nil), + Entry("only VC Port is updated", nil, "some-port"), + Entry("both VC PNID and Port are updated", "some-pnid", "some-port"), + Entry("neither VC PNID and Port are updated", nil, nil), + ) + }) + }) +} + +var _ = Describe("ConfigMapToProviderConfig", func() { + + var ( + providerCreds *credentials.VSphereVMProviderCredentials + providerConfigIn *config.VSphereVMProviderConfig + configMap *corev1.ConfigMap + ) + + BeforeEach(func() { + providerCreds = &credentials.VSphereVMProviderCredentials{Username: "username", Password: "password"} + providerConfigIn = &config.VSphereVMProviderConfig{ + VcPNID: "my-vc.vmware.com", + VcPort: "433", + VcCreds: providerCreds, + Datacenter: "datacenter-42", + StorageClassRequired: false, + UseInventoryAsContentSource: false, + CAFilePath: "/etc/pki/tls/certs/ca-bundle.crt", + InsecureSkipTLSVerify: false, + ResourcePool: "resourcepool-42", + Folder: "folder-42", + Datastore: "/DC0/datastore/LocalDS_0", + } + }) + + JustBeforeEach(func() { + configMap = config.ProviderConfigToConfigMap("dummy-ns", providerConfigIn, "dummy-secrets") + }) + + It("provider config is correctly extracted from the ConfigMap", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.VcPNID).To(Equal(configMap.Data["VcPNID"])) + Expect(providerConfig.VcPort).To(Equal(configMap.Data["VcPort"])) + }) + + Context("when VcPNID is unset in configMap", func() { + It("return an error", func() { + delete(configMap.Data, "VcPNID") + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).To(HaveOccurred()) + Expect(providerConfig).To(BeNil()) + }) + }) + + Context("StorageClassRequired", func() { + It("StorageClassRequired is unset in configMap", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.StorageClassRequired).To(BeFalse()) + }) + + Context("StorageClassRequired is set in configMap", func() { + BeforeEach(func() { + providerConfigIn.StorageClassRequired = true + }) + + It("StorageClassRequired is true in config", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.StorageClassRequired).To(BeTrue()) + }) + }) + }) + + Describe("Tests for TLS configuration", func() { + + Context("when no TLS configuration is specified", func() { + It("defaults to using TLS with the system root CA", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).NotTo(HaveOccurred()) + + Expect(providerConfig.InsecureSkipTLSVerify).To(BeFalse()) + Expect(providerConfig.CAFilePath).To(Equal("/etc/pki/tls/certs/ca-bundle.crt")) + }) + }) + + Context("when the config chooses to ignore TLS verification", func() { + BeforeEach(func() { + providerConfigIn.InsecureSkipTLSVerify = true + }) + + It("sets the insecure flag in the provider config", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).NotTo(HaveOccurred()) + Expect(providerConfig.InsecureSkipTLSVerify).To(BeTrue()) + }) + }) + + Context("when the TLS settings in the Config do not pars", func() { + It("returns an error when parsing the ConfigMa", func() { + configMap.Data["InsecureSkipTLSVerify"] = "bogus" + _, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when the config chooses to use TLS verification and overrides the CA file path", func() { + BeforeEach(func() { + providerConfigIn.CAFilePath = "/etc/a/new/ca/bundle.crt" + }) + + It("uses the new CA path", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.CAFilePath).To(Equal("/etc/a/new/ca/bundle.crt")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/constants/constants.go b/pkg/vmprovider/providers/vsphere2/constants/constants.go new file mode 100644 index 000000000..ef26052b7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/constants/constants.go @@ -0,0 +1,137 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package constants + +import ( + "github.com/vmware-tanzu/vm-operator/pkg" +) + +const ( + ExtraConfigTrue = "TRUE" + ExtraConfigFalse = "FALSE" + ExtraConfigUnset = "" + ExtraConfigGuestInfoPrefix = "guestinfo." + + // VCVMAnnotation Annotation placed on the VM. + VCVMAnnotation = "Virtual Machine managed by the vSphere Virtual Machine service" + + // ManagedByExtensionKey and ManagedByExtensionType represent the ManagedBy field on the VM. + // Historically, this field was used to differentiate VM Service managed VMs from traditional ones. + ManagedByExtensionKey = "com.vmware.vcenter.wcp" + ManagedByExtensionType = "VirtualMachine" + + // VSphereCustomizationBypassKey Annotation to skip applying VMware Tools Guest Customization. + VSphereCustomizationBypassKey = pkg.VMOperatorKey + "/vsphere-customization" + VSphereCustomizationBypassDisable = "disable" + + // VMOperatorV1Alpha1ExtraConfigKey Special ExtraConfig key for v1alpha1 images. + VMOperatorV1Alpha1ExtraConfigKey = "guestinfo.vmservice.defer-cloud-init" + VMOperatorV1Alpha1ConfigReady = "ready" + VMOperatorV1Alpha1ConfigEnabled = "enabled" + + // GOSCPendingExtraConfigKey and GOSCIgnoreToolsCheckExtraConfigKey are GOSC Related ExtraConfig keys. + GOSCPendingExtraConfigKey = "tools.deployPkg.fileName" + GOSCIgnoreToolsCheckExtraConfigKey = "vmware.tools.gosc.ignoretoolscheck" + + // EnableDiskUUIDExtraConfigKey Enable UUID ExtraConfig key. + EnableDiskUUIDExtraConfigKey = "disk.enableUUID" + + // MMPowerOffVMExtraConfigKey ExtraConfig key to enable DRS to powerOff VMs when underlying host enters into + // maintenance mode. This is to ensure the maintenance mode workflow is consistent for VMs with vGPU/DDPIO devices. + MMPowerOffVMExtraConfigKey = "maintenance.vm.evacuation.poweroff" + + // NetPlanVersion points to the version used for Network config. + // For more information, please see https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v2.html + NetPlanVersion = 2 + + // VMImageCLVersionAnnotation VirtualMachineImage annotation to cache the last fetched version. + VMImageCLVersionAnnotation = pkg.VMOperatorKey + "/content-library-version" + // VMImageCLVersionAnnotationVersion is the version of the VMImageCLVersionAnnotation for the VirtualMachineImage. + VMImageCLVersionAnnotationVersion = 1 + + PCIPassthruMMIOOverrideAnnotation = pkg.VMOperatorKey + "/pci-passthru-64bit-mmio-size" + PCIPassthruMMIOExtraConfigKey = "pciPassthru.use64bitMMIO" //nolint:gosec + PCIPassthruMMIOSizeExtraConfigKey = "pciPassthru.64bitMMIOSizeGB" //nolint:gosec + PCIPassthruMMIOSizeDefault = "512" + + // MinSupportedHWVersionForPVC is the supported virtual hardware version for persistent volumes. + MinSupportedHWVersionForPVC = 15 + // MinSupportedHWVersionForPCIPassthruDevices is the supported virtual hardware version for NVidia PCI devices. + MinSupportedHWVersionForPCIPassthruDevices = 17 + + // FirmwareOverrideAnnotation is the annotation key used for firmware override. + FirmwareOverrideAnnotation = pkg.VMOperatorKey + "/firmware" + + CloudInitTypeAnnotation = pkg.VMOperatorKey + "/cloudinit-type" + CloudInitTypeValueCloudInitPrep = "cloudinitprep" + CloudInitTypeValueGuestInfo = "guestinfo" + + CloudInitGuestInfoMetadata = "guestinfo.metadata" + CloudInitGuestInfoMetadataEncoding = "guestinfo.metadata.encoding" + CloudInitGuestInfoUserdata = "guestinfo.userdata" + CloudInitGuestInfoUserdataEncoding = "guestinfo.userdata.encoding" + + // InstanceStoragePVCNamePrefix prefix of auto-generated PVC names. + InstanceStoragePVCNamePrefix = "instance-pvc-" + // InstanceStorageLabelKey identifies resources related to instance storage. + // The primary purpose of this label is to identify instance storage resources such as + // PVCs and CNSNodeVMAttachments but not for List and kubectl-get of VM resources. + InstanceStorageLabelKey = "vmoperator.vmware.com/instance-storage-resource" + // InstanceStoragePVCsBoundAnnotationKey annotation key used to set bound state of all instance storage PVCs. + InstanceStoragePVCsBoundAnnotationKey = "vmoperator.vmware.com/instance-storage-pvcs-bound" + // InstanceStoragePVPlacementErrorAnnotationKey annotation key to set PV creation error. + // CSI reference to this annotation where it is defined: + // https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/pkg/syncer/k8scloudoperator/placement.go + InstanceStoragePVPlacementErrorAnnotationKey = "failure-domain.beta.vmware.com/storagepool" + // InstanceStorageSelectedNodeMOIDAnnotationKey value corresponds to MOID of ESXi node that is elected to place instance storage volumes. + InstanceStorageSelectedNodeMOIDAnnotationKey = "vmoperator.vmware.com/instance-storage-selected-node-moid" + // InstanceStorageSelectedNodeAnnotationKey value corresponds to FQDN of ESXi node that is elected to place instance storage volumes. + InstanceStorageSelectedNodeAnnotationKey = "vmoperator.vmware.com/instance-storage-selected-node" + // KubernetesSelectedNodeAnnotationKey annotation key to set selected node on PVC. + KubernetesSelectedNodeAnnotationKey = "volume.kubernetes.io/selected-node" + // InstanceStoragePVPlacementErrorPrefix indicates prefix of error value. + InstanceStoragePVPlacementErrorPrefix = "FAILED_" + // InstanceStorageNotEnoughResErr is an error constant to indicate not enough resources. + InstanceStorageNotEnoughResErr = "FAILED_PLACEMENT-NotEnoughResources" + // InstanceStorageVDiskID vDisk ID for instance storage volume. + InstanceStorageVDiskID = "cc737f33-2aa3-4594-aa60-df7d6d4cb984" + + // XsiNamespace indicates the XML scheme instance namespace. + XsiNamespace = "http://www.w3.org/2001/XMLSchema-instance" + // ConfigSpecProviderXML indicates XML as the config spec transport type for virtual machine deployment. + ConfigSpecProviderXML = "XML" + + // V1alpha1FirstIP is an alias for versioned templating function V1alpha1_FirstIP. + V1alpha1FirstIP = "V1alpha1_FirstIP" + // V1alpha1FirstNicMacAddr is an alias for versioned templating function V1alpha1_FirstNicMacAddr. + V1alpha1FirstNicMacAddr = "V1alpha1_FirstNicMacAddr" + // V1alpha1FirstIPFromNIC is an alias for versioned templating function V1alpha1_FirstIPFromNIC. + V1alpha1FirstIPFromNIC = "V1alpha1_FirstIPFromNIC" + // V1alpha1IPsFromNIC is an alias for versioned templating function V1alpha1_IPsFromNIC. + V1alpha1IPsFromNIC = "V1alpha1_IPsFromNIC" + // V1alpha1FormatIP is an alias for versioned templating function V1alpha1_FormatIP. + V1alpha1FormatIP = "V1alpha1_FormatIP" + // V1alpha1IP is an alias for versioned templating function V1alpha1_IP. + V1alpha1IP = "V1alpha1_IP" + // V1alpha1SubnetMask is an alias for versioned templating function V1alpha1_SubnetMask. + V1alpha1SubnetMask = "V1alpha1_SubnetMask" + // V1alpha1FormatNameservers is an alias for versioned templating function V1alpha1_FormatNameservers. + V1alpha1FormatNameservers = "V1alpha1_FormatNameservers" + // V1alpha2FirstIP is an alias for versioned templating function V1alpha2_FirstIP. + V1alpha2FirstIP = "V1alpha2_FirstIP" + // V1alpha2FirstNicMacAddr is an alias for versioned templating function V1alpha2_FirstNicMacAddr. + V1alpha2FirstNicMacAddr = "V1alpha2_FirstNicMacAddr" + // V1alpha2FirstIPFromNIC is an alias for versioned templating function V1alpha2_FirstIPFromNIC. + V1alpha2FirstIPFromNIC = "V1alpha2_FirstIPFromNIC" + // V1alpha2IPsFromNIC is an alias for versioned templating function V1alpha2_IPsFromNIC. + V1alpha2IPsFromNIC = "V1alpha2_IPsFromNIC" + // V1alpha2FormatIP is an alias for versioned templating function V1alpha2_FormatIP. + V1alpha2FormatIP = "V1alpha2_FormatIP" + // V1alpha2IP is an alias for versioned templating function V1alpha2_IP. + V1alpha2IP = "V1alpha2_IP" + // V1alpha2SubnetMask is an alias for versioned templating function V1alpha2_SubnetMask. + V1alpha2SubnetMask = "V1alpha2_SubnetMask" + // V1alpha2FormatNameservers is an alias for versioned templating function V1alpha2_FormatNameservers. + V1alpha2FormatNameservers = "V1alpha2_FormatNameservers" +) diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go new file mode 100644 index 000000000..7dc3b9f54 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go @@ -0,0 +1,10 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var log = logf.Log.WithName("vsphere").WithName("contentlibrary") diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go new file mode 100644 index 000000000..be2ff212d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go @@ -0,0 +1,379 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + "strconv" + "time" + + k8serrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/soap" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type Provider interface { + GetLibraryItems(ctx context.Context, clUUID string) ([]library.Item, error) + GetLibraryItem(ctx context.Context, libraryUUID, itemName string, + notFoundReturnErr bool) (*library.Item, error) + GetLibraryItemID(ctx context.Context, itemUUID string) (*library.Item, error) + ListLibraryItems(ctx context.Context, libraryUUID string) ([]string, error) + UpdateLibraryItem(ctx context.Context, itemID, newName string, newDescription *string) error + RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item *library.Item) (*ovf.Envelope, error) + RetrieveOvfEnvelopeByLibraryItemID(ctx context.Context, itemID string) (*ovf.Envelope, error) + + // TODO: Testing only. Remove these from this file. + CreateLibraryItem(ctx context.Context, libraryItem library.Item, path string) error +} + +type provider struct { + libMgr *library.Manager + retryInterval time.Duration +} + +const ( + EnvContentLibAPIWaitSecs = "CONTENT_API_WAIT_SECS" // BMV: Investigate if setting this to 1 actually reduces the integration test time. + DefaultContentLibAPIWaitSecs = 5 +) + +func IsSupportedDeployType(t string) bool { + switch t { + case library.ItemTypeVMTX, library.ItemTypeOVF: + // Keep in sync with what cloneVMFromContentLibrary() handles. + return true + default: + return false + } +} + +func NewProvider(restClient *rest.Client) Provider { + waitSeconds, err := strconv.Atoi(os.Getenv(EnvContentLibAPIWaitSecs)) + if err != nil || waitSeconds < 1 { + waitSeconds = DefaultContentLibAPIWaitSecs + } + + return NewProviderWithWaitSec(restClient, waitSeconds) +} + +func NewProviderWithWaitSec(restClient *rest.Client, waitSeconds int) Provider { + return &provider{ + libMgr: library.NewManager(restClient), + retryInterval: time.Duration(waitSeconds) * time.Second, + } +} + +func (cs *provider) ListLibraryItems(ctx context.Context, libraryUUID string) ([]string, error) { + logger := log.WithValues("libraryUUID", libraryUUID) + itemList, err := cs.libMgr.ListLibraryItems(ctx, libraryUUID) + if err != nil { + if lib.IsNotFoundError(err) { + logger.Error(err, "cannot list items from content library that does not exist") + return nil, nil + } + return nil, err + } + return itemList, err +} + +func (cs *provider) GetLibraryItems(ctx context.Context, libraryUUID string) ([]library.Item, error) { + logger := log.WithValues("libraryUUID", libraryUUID) + itemList, err := cs.libMgr.ListLibraryItems(ctx, libraryUUID) + if err != nil { + if lib.IsNotFoundError(err) { + logger.Error(err, "cannot list items from content library that does not exist") + return nil, nil + } + return nil, err + } + + // best effort to get content library items. + resErrs := make([]error, 0) + items := make([]library.Item, 0) + for _, itemID := range itemList { + item, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + resErrs = append(resErrs, err) + logger.Error(err, "get library item failed", "itemID", itemID) + continue + } + items = append(items, *item) + } + + return items, k8serrors.NewAggregate(resErrs) +} + +func (cs *provider) GetLibraryItem(ctx context.Context, libraryUUID, itemName string, + notFoundReturnErr bool) (*library.Item, error) { + itemIDs, err := cs.libMgr.FindLibraryItems(ctx, library.FindItem{LibraryID: libraryUUID, Name: itemName}) + if err != nil { + return nil, errors.Wrapf(err, "failed to find image: %s", itemName) + } + + if len(itemIDs) == 0 { + if notFoundReturnErr { + return nil, errors.Errorf("no library item named: %s", itemName) + } + return nil, nil + } + if len(itemIDs) != 1 { + return nil, errors.Errorf("multiple library items named: %s", itemName) + } + + item, err := cs.libMgr.GetLibraryItem(ctx, itemIDs[0]) + if err != nil { + return nil, errors.Wrapf(err, "failed to get library item: %s", itemName) + } + + return item, nil +} + +func (cs *provider) GetLibraryItemID(ctx context.Context, itemUUID string) (*library.Item, error) { + item, err := cs.libMgr.GetLibraryItem(ctx, itemUUID) + if err != nil { + return nil, errors.Wrapf(err, "failed to find image: %s", itemUUID) + } + + return item, nil +} + +// RetrieveOvfEnvelopeByLibraryItemID retrieves the OVF Envelope by the given library item ID. +func (cs *provider) RetrieveOvfEnvelopeByLibraryItemID(ctx context.Context, itemID string) (*ovf.Envelope, error) { + libItem, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + return nil, err + } + + if libItem == nil || libItem.Type != library.ItemTypeOVF { + log.Error(nil, "empty or non OVF library item type, skipping", "itemID", itemID) + // No need to return the error here to avoid unnecessary reconciliation. + return nil, nil + } + + return cs.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem) +} + +func readerFromURL(ctx context.Context, c *rest.Client, url *url.URL) (io.ReadCloser, error) { + p := soap.DefaultDownload + readerStream, _, err := c.Download(ctx, url, &p) + if err != nil { + // Log message used by VMC LINT. Refer to before making changes + log.Error(err, "Error occurred when downloading file", "url", url) + return nil, err + } + + return readerStream, nil +} + +// RetrieveOvfEnvelopeFromLibraryItem downloads the supported file from content library. +// parses the downloaded ovf and returns the OVF Envelope descriptor for consumption. +func (cs *provider) RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item *library.Item) (*ovf.Envelope, error) { + // Create a download session for the file referred to by item id. + sessionID, err := cs.libMgr.CreateLibraryItemDownloadSession(ctx, library.Session{LibraryItemID: item.ID}) + if err != nil { + return nil, err + } + + logger := log.WithValues("sessionID", sessionID, "itemID", item.ID, "itemName", item.Name) + logger.V(4).Info("download session for item created") + + defer func() { + if err := cs.libMgr.DeleteLibraryItemDownloadSession(ctx, sessionID); err != nil { + logger.Error(err, "Error deleting download session") + } + }() + + // Download ovf from the library item. + fileURL, err := cs.generateDownloadURLForLibraryItem(ctx, logger, sessionID, item) + if err != nil { + return nil, err + } + + downloadedFileContent, err := readerFromURL(ctx, cs.libMgr.Client, fileURL) + if err != nil { + logger.Error(err, "error downloading file from library item") + return nil, err + } + + logger.V(4).Info("downloaded library item") + defer func() { + _ = downloadedFileContent.Close() + }() + + envelope, err := ovf.Unmarshal(downloadedFileContent) + if err != nil { + logger.Error(err, "error parsing the OVF envelope") + return nil, nil + } + + return envelope, nil +} + +// UpdateLibraryItem updates the content library item's name and description. +func (cs *provider) UpdateLibraryItem(ctx context.Context, itemID, newName string, newDescription *string) error { + log.Info("Updating Library Item", "itemID", itemID, + "newName", newName, "newDescription", newDescription) + + item, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + log.Error(err, "error getting library item") + return err + } + + if newName != "" { + item.Name = newName + } + if newDescription != nil { + item.Description = newDescription + } + + return cs.libMgr.UpdateLibraryItem(ctx, item) +} + +// Only used in testing. +func (cs *provider) CreateLibraryItem(ctx context.Context, libraryItem library.Item, path string) error { + log.Info("Creating Library Item", "item", libraryItem, "path", path) + + itemID, err := cs.libMgr.CreateLibraryItem(ctx, libraryItem) + if err != nil { + return err + } + + sessionID, err := cs.libMgr.CreateLibraryItemUpdateSession(ctx, library.Session{LibraryItemID: itemID}) + if err != nil { + return err + } + + // Update Library item with library file "ovf" + uploadFunc := func(c *rest.Client, path string) error { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + + fi, err := f.Stat() + if err != nil { + return err + } + + info := library.UpdateFile{ + Name: filepath.Base(path), + SourceType: "PUSH", + Size: fi.Size(), + } + + update, err := cs.libMgr.AddLibraryItemFile(ctx, sessionID, info) + if err != nil { + return err + } + + u, err := url.Parse(update.UploadEndpoint.URI) + if err != nil { + return err + } + + p := soap.DefaultUpload + p.ContentLength = info.Size + + return c.Upload(ctx, f, u, &p) + } + + if err = uploadFunc(cs.libMgr.Client, path); err != nil { + return err + } + + return cs.libMgr.CompleteLibraryItemUpdateSession(ctx, sessionID) +} + +// generateDownloadURLForLibraryItem downloads the file from content library in 3 steps: +// 1. list the available files and downloads only the ovf files based on filename suffix +// 2. prepare the download session and fetch the url to be used for download +// 3. download the file. +func (cs *provider) generateDownloadURLForLibraryItem( + ctx context.Context, + logger logr.Logger, + sessionID string, + item *library.Item) (*url.URL, error) { + + // List the files available for download in the library item. + files, err := cs.libMgr.ListLibraryItemDownloadSessionFile(ctx, sessionID) + if err != nil { + return nil, err + } + + var fileToDownload string + for _, file := range files { + logger.V(4).Info("Library Item file", "fileName", file.Name) + if ext := filepath.Ext(file.Name); ext != "" && IsSupportedDeployType(ext[1:]) { + fileToDownload = file.Name + break + } + } + if fileToDownload == "" { + return nil, errors.Errorf("No files with supported deploy type are available for download for %s", item.ID) + } + + _, err = cs.libMgr.PrepareLibraryItemDownloadSessionFile(ctx, sessionID, fileToDownload) + if err != nil { + return nil, err + } + + logger.V(4).Info("request posted to prepare file", "fileToDownload", fileToDownload) + + // Content library api to prepare a file for download guarantees eventual end state of either + // ERROR or PREPARED in order to avoid posting too many requests to the api. + var fileURL string + err = wait.PollUntilContextCancel(ctx, cs.retryInterval, true, func(_ context.Context) (bool, error) { + downloadSessResp, err := cs.libMgr.GetLibraryItemDownloadSession(ctx, sessionID) + if err != nil { + return false, err + } + + if downloadSessResp.ErrorMessage != nil { + return false, downloadSessResp.ErrorMessage + } + + info, err := cs.libMgr.GetLibraryItemDownloadSessionFile(ctx, sessionID, fileToDownload) + if err != nil { + return false, err + } + + if info.Status == "ERROR" { + // Log message used by VMC LINT. Refer to before making changes + return false, errors.Errorf("Error occurred preparing file for download %v", info.ErrorMessage) + } + + if info.Status != "PREPARED" { + return false, nil + } + + if info.DownloadEndpoint == nil { + return false, errors.Errorf("Prepared file for download does not have endpoint") + } + + fileURL = info.DownloadEndpoint.URI + log.V(4).Info("Downloaded file", "fileURL", fileURL) + return true, nil + }) + + if err != nil { + return nil, err + } + + return url.Parse(fileURL) +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go new file mode 100644 index 000000000..2055b76eb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ContentLibrary Provider", clTests) +} + +var suite = builder.NewTestSuite() + +func TestContentLibrary(t *testing.T) { + suite.Register(t, "vSphere Provider ContentLibrary Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go new file mode 100644 index 000000000..968eac039 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go @@ -0,0 +1,139 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + "os" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vapi/library" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func clTests() { + Describe("Content Library", func() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + + clProvider contentlibrary.Provider + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + testConfig.WithContentLibrary = true + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + clProvider = contentlibrary.NewProvider(ctx.RestClient) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("when items are present in library", func() { + + It("List items id in library", func() { + items, err := clProvider.GetLibraryItems(ctx, ctx.ContentLibraryID) + Expect(err).ToNot(HaveOccurred()) + Expect(items).ToNot(BeEmpty()) + }) + + It("Does not return error when library does not exist", func() { + items, err := clProvider.GetLibraryItems(ctx, "dummy-cl") + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(BeEmpty()) + }) + + It("Does not return error when item name is invalid when notFoundReturnErr is set to false", func() { + item, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, "dummy-name", true) + Expect(err).To(HaveOccurred()) + Expect(item).To(BeNil()) + + item, err = clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, "dummy-name", false) + Expect(err).NotTo(HaveOccurred()) + Expect(item).To(BeNil()) + }) + + It("Gets items and returns OVF", func() { + item, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, ctx.ContentLibraryImageName, true) + Expect(err).ToNot(HaveOccurred()) + Expect(item).ToNot(BeNil()) + + ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, item) + Expect(err).ToNot(HaveOccurred()) + Expect(ovfEnvelope).ToNot(BeNil()) + }) + }) + + Context("when items are not present in library", func() { + + Context("when invalid item id is passed", func() { + + It("returns an error creating a download session", func() { + libItem := &library.Item{ + Name: "fakeItem", + Type: "ovf", + LibraryID: "fakeID", + } + + ovf, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("404 Not Found")) + Expect(ovf).To(BeNil()) + }) + }) + }) + + Context("called with an OVF that is invalid", func() { + var ovfPath string + + AfterEach(func() { + if ovfPath != "" { + Expect(os.Remove(ovfPath)).To(Succeed()) + } + }) + + It("does not return error", func() { + ovf, err := os.CreateTemp("", "fake-*.ovf") + Expect(err).NotTo(HaveOccurred()) + ovfPath = ovf.Name() + + ovfInfo, err := ovf.Stat() + Expect(err).NotTo(HaveOccurred()) + + libItemName := strings.Split(ovfInfo.Name(), ".ovf")[0] + libItem := library.Item{ + Name: libItemName, + Type: "ovf", + LibraryID: ctx.ContentLibraryID, + } + + err = clProvider.CreateLibraryItem(ctx, libItem, ovfPath) + Expect(err).NotTo(HaveOccurred()) + + libItem2, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, libItemName, true) + Expect(err).ToNot(HaveOccurred()) + Expect(libItem2).ToNot(BeNil()) + Expect(libItem2.Name).To(Equal(libItem.Name)) + + ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem2) + Expect(err).ToNot(HaveOccurred()) + Expect(ovfEnvelope).To(BeNil()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go new file mode 100644 index 000000000..b4a2fd866 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go @@ -0,0 +1,149 @@ +// Copyright (c) 2019-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + "regexp" + "strconv" + "strings" + + "github.com/vmware/govmomi/ovf" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +var vmxRe = regexp.MustCompile(`vmx-(\d+)`) + +// ParseVirtualHardwareVersion parses the virtual hardware version +// For eg. "vmx-15" returns 15. +func ParseVirtualHardwareVersion(vmxVersion string) int32 { + // obj is the full string and the submatch (\d+) and return a []string with values + obj := vmxRe.FindStringSubmatch(vmxVersion) + if len(obj) != 2 { + return 0 + } + + version, err := strconv.ParseInt(obj[1], 10, 32) + if err != nil { + return 0 + } + + return int32(version) +} + +// UpdateVmiWithOvfEnvelope updates the given vmi object with the content of given OVF envelope. +func UpdateVmiWithOvfEnvelope(vmi client.Object, ovfEnvelope ovf.Envelope) { + var status *vmopv1.VirtualMachineImageStatus + + switch vmi := vmi.(type) { + case *vmopv1.VirtualMachineImage: + status = &vmi.Status + case *vmopv1.ClusterVirtualMachineImage: + status = &vmi.Status + default: + return + } + + if ovfEnvelope.VirtualSystem != nil { + initImageStatusFromOVFVirtualSystem(status, ovfEnvelope.VirtualSystem) + + ovfSystemProps := getVmwareSystemPropertiesFromOvf(ovfEnvelope.VirtualSystem) + if len(ovfSystemProps) > 0 { + annotations := vmi.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + vmi.SetAnnotations(annotations) + } + + for k, v := range ovfSystemProps { + annotations[k] = v + } + } + } +} + +func initImageStatusFromOVFVirtualSystem( + imageStatus *vmopv1.VirtualMachineImageStatus, + ovfVirtualSystem *ovf.VirtualSystem) { + + // Use info from the first product section in the VM image, if one exists. + if product := ovfVirtualSystem.Product; len(product) > 0 { + p := product[0] + + productInfo := &imageStatus.ProductInfo + productInfo.Vendor = p.Vendor + productInfo.Product = p.Product + productInfo.Version = p.Version + productInfo.FullVersion = p.FullVersion + } + + // Use operating system info from the first os section in the VM image, if one exists. + if os := ovfVirtualSystem.OperatingSystem; len(os) > 0 { + o := os[0] + + osInfo := &imageStatus.OSInfo + osInfo.ID = strconv.Itoa(int(o.ID)) + if o.Version != nil { + osInfo.Version = *o.Version + } + if o.OSType != nil { + osInfo.Type = *o.OSType + } + } + + // Use hardware section info from the VM image, if one exists. + if virtualHW := ovfVirtualSystem.VirtualHardware; len(virtualHW) > 0 { + imageStatus.Firmware = getFirmwareType(virtualHW[0]) + + if sys := virtualHW[0].System; sys != nil && sys.VirtualSystemType != nil { + ver := ParseVirtualHardwareVersion(*sys.VirtualSystemType) + if ver != 0 { + imageStatus.HardwareVersion = &ver + } + } + } + + for _, product := range ovfVirtualSystem.Product { + for _, prop := range product.Property { + // Only show user configurable properties + if prop.UserConfigurable != nil && *prop.UserConfigurable { + property := vmopv1.OVFProperty{ + Key: prop.Key, + Type: prop.Type, + Default: prop.Default, + } + imageStatus.OVFProperties = append(imageStatus.OVFProperties, property) + } + } + } +} + +func getVmwareSystemPropertiesFromOvf(ovfVirtualSystem *ovf.VirtualSystem) map[string]string { + properties := make(map[string]string) + + if ovfVirtualSystem != nil { + for _, product := range ovfVirtualSystem.Product { + for _, prop := range product.Property { + if strings.HasPrefix(prop.Key, "vmware-system") { + if prop.Default != nil { + properties[prop.Key] = *prop.Default + } + } + } + } + } + + return properties +} + +// getFirmwareType returns the firmware type (eg: "efi", "bios") present in the virtual hardware section of the OVF. +func getFirmwareType(hardware ovf.VirtualHardwareSection) string { + for _, cfg := range hardware.Config { + if cfg.Key == "firmware" { + return cfg.Value + } + } + return "" +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go new file mode 100644 index 000000000..051f1bb91 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +var _ = Describe("ParseVirtualHardwareVersion", func() { + It("empty hardware string", func() { + vmxHwVersionString := "" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("invalid hardware string", func() { + vmxHwVersionString := "blah" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("valid hardware version string eg. vmx-15", func() { + vmxHwVersionString := "vmx-15" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(Equal(int32(15))) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials.go new file mode 100644 index 000000000..9ac994bf5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials.go @@ -0,0 +1,62 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "context" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" +) + +// VSphereVMProviderCredentials wraps the data needed to login to vCenter. +type VSphereVMProviderCredentials struct { + Username string + Password string +} + +func GetProviderCredentials(client ctrlruntime.Client, namespace, secretName string) (*VSphereVMProviderCredentials, error) { + secret := &corev1.Secret{} + secretKey := types.NamespacedName{Namespace: namespace, Name: secretName} + if err := client.Get(context.Background(), secretKey, secret); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "cannot find secret for provider credentials: %s", secretKey) + } + + var credentials VSphereVMProviderCredentials + credentials.Username = string(secret.Data["username"]) + credentials.Password = string(secret.Data["password"]) + + if credentials.Username == "" || credentials.Password == "" { + return nil, errors.New("vCenter username and password are missing") + } + + return &credentials, nil +} + +func setSecretData(secret *corev1.Secret, credentials *VSphereVMProviderCredentials) { + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + + secret.Data["username"] = []byte(credentials.Username) + secret.Data["password"] = []byte(credentials.Password) +} + +// ProviderCredentialsToSecret returns the Secret for the credentials. +// Testing only. +func ProviderCredentialsToSecret(namespace string, credentials *VSphereVMProviderCredentials, vcCredsSecretName string) *corev1.Secret { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: vcCredsSecretName, + Namespace: namespace, + }, + } + setSecretData(secret, credentials) + + return secret +} diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go new file mode 100644 index 000000000..45cedd1ea --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go @@ -0,0 +1,44 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +var ( + model *simulator.Model + server *simulator.Server + ctx context.Context + tlsTestModel *simulator.Model + tlsServer *simulator.Server + tlsServerCertPath string + tlsServerKeyPath string +) + +var _ = BeforeSuite(func() { + ctx, model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer = test.BeforeSuite() +}) + +var _ = AfterSuite(func() { + test.AfterSuite( + ctx, + model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer) +}) + +func TestCredentials(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Credentials Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go new file mode 100644 index 000000000..8950bdffa --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + + . "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func newSecret(name string, ns string, user string, pass string) (*corev1.Secret, *VSphereVMProviderCredentials) { + creds := &VSphereVMProviderCredentials{ + Username: user, + Password: pass, + } + secret := ProviderCredentialsToSecret(ns, creds, name) + return secret, creds +} + +var _ = Describe("GetProviderCredentials", func() { + + Context("when a good secret exists", func() { + Specify("returns good credentials with no error", func() { + secretIn, credsIn := newSecret("some-name", "some-namespace", "some-user", "some-pass") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).ToNot(HaveOccurred()) + Expect(credsOut).To(Equal(credsIn)) + }) + }) + + Context("when a bad secret exists", func() { + + Context("with empty username", func() { + Specify("returns no credentials with error", func() { + secretIn, _ := newSecret("some-name", "some-namespace", "", "some-pass") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) + + Context("with empty password", func() { + Specify("returns no credentials with error", func() { + secretIn, _ := newSecret("some-name", "some-namespace", "some-user", "") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) + }) + + Context("when no secret exists", func() { + Specify("returns no credentials with error", func() { + client := builder.NewFakeClient() + credsOut, err := GetProviderCredentials(client, "none-namespace", "none-name") + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go b/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go new file mode 100644 index 000000000..698faa7d5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go @@ -0,0 +1,41 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package instancestorage + +import ( + "strings" + + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +// IsPresent checks if VM Spec has instance volumes added to its Volumes list. +func IsPresent(vm *vmopv1.VirtualMachine) bool { + for _, vol := range vm.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil && pvc.InstanceVolumeClaim != nil { + return true + } + } + return false +} + +// FilterVolumes returns instance storage volumes present in VM spec. +func FilterVolumes(vm *vmopv1.VirtualMachine) []vmopv1.VirtualMachineVolume { + var volumes []vmopv1.VirtualMachineVolume + for _, vol := range vm.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil && pvc.InstanceVolumeClaim != nil { + volumes = append(volumes, vol) + } + } + + return volumes +} + +func IsInsufficientQuota(err error) bool { + if apiErrors.IsForbidden(err) && (strings.Contains(err.Error(), "insufficient quota") || strings.Contains(err.Error(), "exceeded quota")) { + return true + } + return false +} diff --git a/pkg/vmprovider/providers/vsphere2/internal/internal.go b/pkg/vmprovider/providers/vsphere2/internal/internal.go new file mode 100644 index 000000000..8a4f68da3 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/internal/internal.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//nolint:revive,stylecheck +package internal + +import ( + "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" +) + +type InvokeFSR_TaskRequest struct { + This types.ManagedObjectReference `xml:"_this"` +} + +type InvokeFSR_TaskResponse struct { + Returnval types.ManagedObjectReference `xml:"returnval"` +} + +type InvokeFSR_TaskBody struct { + Req *InvokeFSR_TaskRequest `xml:"urn:vim25 InvokeFSR_Task,omitempty"` + Res *InvokeFSR_TaskResponse `xml:"InvokeFSR_TaskResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *InvokeFSR_TaskBody) Fault() *soap.Fault { + return b.Fault_ +} + +func InvokeFSR_Task(ctx context.Context, r soap.RoundTripper, req *InvokeFSR_TaskRequest) (*InvokeFSR_TaskResponse, error) { + var reqBody, resBody InvokeFSR_TaskBody + reqBody.Req = req + if err := r.RoundTrip(ctx, &reqBody, &resBody); err != nil { + return nil, err + } + return resBody.Res, nil +} + +func VirtualMachineFSR(ctx context.Context, vm types.ManagedObjectReference, client *vim25.Client) (*object.Task, error) { + req := InvokeFSR_TaskRequest{ + This: vm, + } + res, err := InvokeFSR_Task(ctx, client, &req) + if err != nil { + return nil, err + } + return object.NewTask(client, res.Returnval), nil +} + +type CustomizationCloudinitPrep struct { + types.CustomizationIdentitySettings + + Metadata string `xml:"metadata"` + Userdata string `xml:"userdata,omitempty"` +} diff --git a/pkg/vmprovider/providers/vsphere2/network/gosc.go b/pkg/vmprovider/providers/vsphere2/network/gosc.go new file mode 100644 index 000000000..feb9f9f3a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/gosc.go @@ -0,0 +1,79 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + "net" + + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +func GuestOSCustomization(results NetworkInterfaceResults) ([]vimtypes.CustomizationAdapterMapping, error) { + mappings := make([]vimtypes.CustomizationAdapterMapping, 0, len(results.Results)) + + for _, r := range results.Results { + adapter := vimtypes.CustomizationIPSettings{ + DnsServerList: r.Nameservers, + } + + if r.DHCP4 { + adapter.Ip = &vimtypes.CustomizationDhcpIpGenerator{} + } else { + // GOSC doesn't support multiple IPv4 address per interface so use the first one. Old code + // only ever set one gateway so do the same here too. + for _, ipConfig := range r.IPConfigs { + if !ipConfig.IsIPv4 { + continue + } + + ip, ipNet, err := net.ParseCIDR(ipConfig.IPCIDR) + if err != nil { + return nil, err + } + subnetMask := net.CIDRMask(ipNet.Mask.Size()) + + adapter.Ip = &vimtypes.CustomizationFixedIp{IpAddress: ip.String()} + adapter.SubnetMask = net.IP(subnetMask).String() + adapter.Gateway = []string{ipConfig.Gateway} + break + } + } + + if r.DHCP6 { + adapter.IpV6Spec = &vimtypes.CustomizationIPSettingsIpV6AddressSpec{ + Ip: []vimtypes.BaseCustomizationIpV6Generator{ + &vimtypes.CustomizationDhcpIpV6Generator{}, + }, + } + } else { + for _, ipConfig := range r.IPConfigs { + if ipConfig.IsIPv4 { + continue + } + + ip, ipNet, err := net.ParseCIDR(ipConfig.IPCIDR) + if err != nil { + return nil, err + } + ones, _ := ipNet.Mask.Size() + + if adapter.IpV6Spec == nil { + adapter.IpV6Spec = &vimtypes.CustomizationIPSettingsIpV6AddressSpec{} + } + adapter.IpV6Spec.Ip = append(adapter.IpV6Spec.Ip, &vimtypes.CustomizationFixedIpV6{ + IpAddress: ip.String(), + SubnetMask: int32(ones), + }) + adapter.IpV6Spec.Gateway = append(adapter.IpV6Spec.Gateway, ipConfig.Gateway) + } + } + + mappings = append(mappings, vimtypes.CustomizationAdapterMapping{ + MacAddress: r.MacAddress, + Adapter: adapter, + }) + } + + return mappings, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/network/gosc_test.go b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go new file mode 100644 index 000000000..1eea859a8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +var _ = Describe("GOSC", func() { + const ( + macAddr1 = "50-8A-80-9D-28-22" + + ipv4Gateway = "192.168.1.1" + ipv4 = "192.168.1.10" + ipv4CIDR = ipv4 + "/24" + ipv6Gateway = "fd8e:b5a0:f172:123::1" + ipv6 = "fd8e:b5a0:f172:123::f" + ipv6Subnet = 48 + + dnsServer1 = "9.9.9.9" + ) + + Context("GuestOSCustomization", func() { + + var ( + results network.NetworkInterfaceResults + adapterMappings []types.CustomizationAdapterMapping + err error + ) + + BeforeEach(func() { + results.Results = nil + }) + + JustBeforeEach(func() { + adapterMappings, err = network.GuestOSCustomization(results) + }) + + Context("IPv4/6 Static adapter", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + IPCIDR: ipv4CIDR, + IsIPv4: true, + Gateway: ipv4Gateway, + }, + { + IPCIDR: ipv6 + fmt.Sprintf("/%d", ipv6Subnet), + IsIPv4: false, + Gateway: ipv6Gateway, + }, + }, + MacAddress: macAddr1, + Name: "eth0", + DHCP4: false, + DHCP6: false, + MTU: 1500, // AFAIK not supported via GOSC + Nameservers: []string{dnsServer1}, + Routes: nil, // AFAIK not supported via GOSC + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(adapterMappings).To(HaveLen(1)) + mapping := adapterMappings[0] + + adapter := mapping.Adapter + Expect(mapping.MacAddress).To(Equal(macAddr1)) + Expect(adapter.Gateway).To(Equal([]string{ipv4Gateway})) + Expect(adapter.SubnetMask).To(Equal("255.255.255.0")) + Expect(adapter.DnsServerList).To(Equal([]string{dnsServer1})) + Expect(adapter.Ip).To(BeAssignableToTypeOf(&types.CustomizationFixedIp{})) + fixedIP := adapter.Ip.(*types.CustomizationFixedIp) + Expect(fixedIP.IpAddress).To(Equal(ipv4)) + + ipv6Spec := adapter.IpV6Spec + Expect(ipv6Spec).ToNot(BeNil()) + Expect(ipv6Spec.Gateway).To(Equal([]string{ipv6Gateway})) + Expect(ipv6Spec.Ip).To(HaveLen(1)) + Expect(ipv6Spec.Ip[0]).To(BeAssignableToTypeOf(&types.CustomizationFixedIpV6{})) + addressSpec := ipv6Spec.Ip[0].(*types.CustomizationFixedIpV6) + Expect(addressSpec.IpAddress).To(Equal(ipv6)) + Expect(addressSpec.SubnetMask).To(BeEquivalentTo(ipv6Subnet)) + }) + }) + + Context("IPv4/6 DHCP", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + Name: "eth0", + DHCP4: true, + DHCP6: true, + MTU: 1500, // AFAIK not support via GOSC + Nameservers: []string{dnsServer1}, + Routes: nil, // AFAIK not support via GOSC + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(adapterMappings).To(HaveLen(1)) + mapping := adapterMappings[0] + + adapter := mapping.Adapter + Expect(mapping.MacAddress).To(Equal(macAddr1)) + Expect(adapter.Gateway).To(BeEmpty()) + Expect(adapter.SubnetMask).To(BeEmpty()) + + Expect(adapter.Ip).To(BeAssignableToTypeOf(&types.CustomizationDhcpIpGenerator{})) + ipv6Spec := adapter.IpV6Spec + Expect(ipv6Spec).ToNot(BeNil()) + Expect(ipv6Spec.Ip).To(HaveLen(1)) + Expect(ipv6Spec.Ip[0]).To(BeAssignableToTypeOf(&types.CustomizationDhcpIpV6Generator{})) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/netplan.go b/pkg/vmprovider/providers/vsphere2/network/netplan.go new file mode 100644 index 000000000..71ca80b45 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/netplan.go @@ -0,0 +1,103 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + "strings" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +// Netplan representation described in https://via.vmw.com/cloud-init-netplan // FIXME: 404. +type Netplan struct { + Version int `yaml:"version,omitempty"` + Ethernets map[string]NetplanEthernet `yaml:"ethernets,omitempty"` +} + +type NetplanEthernet struct { + Match NetplanEthernetMatch `yaml:"match,omitempty"` + SetName string `yaml:"set-name,omitempty"` + Dhcp4 bool `yaml:"dhcp4,omitempty"` + Dhcp6 bool `yaml:"dhcp6,omitempty"` + Addresses []string `yaml:"addresses,omitempty"` + Gateway4 string `yaml:"gateway4,omitempty"` + Gateway6 string `yaml:"gateway6,omitempty"` + MTU int64 `yaml:"mtu,omitempty"` + Nameservers NetplanEthernetNameserver `yaml:"nameservers,omitempty"` + Routes []NetplanEthernetRoute `yaml:"routes,omitempty"` +} + +type NetplanEthernetMatch struct { + MacAddress string `yaml:"macaddress,omitempty"` +} + +type NetplanEthernetNameserver struct { + Addresses []string `yaml:"addresses,omitempty"` + Search []string `yaml:"search,omitempty"` +} + +type NetplanEthernetRoute struct { + To string `yaml:"to"` + Via string `yaml:"via"` + Metric int32 `yaml:"metric,omitempty"` +} + +func NetPlanCustomization(result NetworkInterfaceResults) (*Netplan, error) { + netPlan := &Netplan{ + Version: constants.NetPlanVersion, + Ethernets: make(map[string]NetplanEthernet), + } + + for _, r := range result.Results { + npEth := NetplanEthernet{ + Match: NetplanEthernetMatch{ + MacAddress: NormalizeNetplanMac(r.MacAddress), + }, + SetName: r.Name, + MTU: r.MTU, + Nameservers: NetplanEthernetNameserver{ + Addresses: r.Nameservers, + Search: r.SearchDomains, + }, + } + + npEth.Dhcp4 = r.DHCP4 + npEth.Dhcp6 = r.DHCP6 + + if !npEth.Dhcp4 { + for _, ipConfig := range r.IPConfigs { + if ipConfig.IsIPv4 { + if npEth.Gateway4 == "" { + npEth.Gateway4 = ipConfig.Gateway + } + npEth.Addresses = append(npEth.Addresses, ipConfig.IPCIDR) + } + } + } + if !npEth.Dhcp6 { + for _, ipConfig := range r.IPConfigs { + if !ipConfig.IsIPv4 { + if npEth.Gateway6 == "" { + npEth.Gateway6 = ipConfig.Gateway + } + npEth.Addresses = append(npEth.Addresses, ipConfig.IPCIDR) + } + } + } + + for _, route := range r.Routes { + npEth.Routes = append(npEth.Routes, NetplanEthernetRoute(route)) + } + + netPlan.Ethernets[npEth.SetName] = npEth + } + + return netPlan, nil +} + +// NormalizeNetplanMac normalizes the mac address format to one compatible with netplan. +func NormalizeNetplanMac(mac string) string { + mac = strings.ReplaceAll(mac, "-", ":") + return strings.ToLower(mac) +} diff --git a/pkg/vmprovider/providers/vsphere2/network/netplan_test.go b/pkg/vmprovider/providers/vsphere2/network/netplan_test.go new file mode 100644 index 000000000..5e3e68375 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/netplan_test.go @@ -0,0 +1,143 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +var _ = Describe("Netplan", func() { + const ( + ifName = "eth0" + macAddr1 = "50-8A-80-9D-28-22" + macAddr1Norm = "50:8a:80:9d:28:22" + ipv4Gateway = "192.168.1.1" + ipv4 = "192.168.1.10" + ipv4CIDR = ipv4 + "/24" + ipv6Gateway = "fd8e:b5a0:f172:123::1" + ipv6 = "fd8e:b5a0:f172:123::f" + ipv6Subnet = 48 + dnsServer1 = "9.9.9.9" + searchDomain1 = "foobar.local" + ) + + Context("NetPlanCustomization", func() { + + var ( + results network.NetworkInterfaceResults + netplan *network.Netplan + err error + ) + + BeforeEach(func() { + results.Results = nil + netplan = nil + }) + + JustBeforeEach(func() { + netplan, err = network.NetPlanCustomization(results) + }) + + Context("IPv4/6 Static adapter", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + IPCIDR: ipv4CIDR, + IsIPv4: true, + Gateway: ipv4Gateway, + }, + { + IPCIDR: ipv6 + fmt.Sprintf("/%d", ipv6Subnet), + IsIPv4: false, + Gateway: ipv6Gateway, + }, + }, + MacAddress: macAddr1, + Name: ifName, + DHCP4: false, + DHCP6: false, + MTU: 1500, + Nameservers: []string{dnsServer1}, + SearchDomains: []string{searchDomain1}, + Routes: []network.NetworkInterfaceRoute{ + { + To: "185.107.56.59", + Via: "10.1.1.1", + Metric: 42, + }, + }, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(netplan).ToNot(BeNil()) + Expect(netplan.Version).To(Equal(constants.NetPlanVersion)) + + Expect(netplan.Ethernets).To(HaveLen(1)) + Expect(netplan.Ethernets).To(HaveKey(ifName)) + + np := netplan.Ethernets[ifName] + Expect(np.Match.MacAddress).To(Equal(macAddr1Norm)) + Expect(np.SetName).To(Equal(ifName)) + Expect(np.Dhcp4).To(BeFalse()) + Expect(np.Dhcp6).To(BeFalse()) + Expect(np.Gateway4).To(Equal(ipv4Gateway)) + Expect(np.Gateway6).To(Equal(ipv6Gateway)) + Expect(np.MTU).To(BeEquivalentTo(1500)) + Expect(np.Nameservers.Addresses).To(Equal([]string{dnsServer1})) + Expect(np.Nameservers.Search).To(Equal([]string{searchDomain1})) + Expect(np.Routes).To(HaveLen(1)) + route := np.Routes[0] + Expect(route.To).To(Equal("185.107.56.59")) + Expect(route.Via).To(Equal("10.1.1.1")) + Expect(route.Metric).To(BeEquivalentTo(42)) + }) + }) + + Context("IPv4/6 DHCP", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + Name: "eth0", + DHCP4: true, + DHCP6: true, + MTU: 9000, + Nameservers: []string{dnsServer1}, + SearchDomains: []string{searchDomain1}, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(netplan).ToNot(BeNil()) + Expect(netplan.Version).To(Equal(constants.NetPlanVersion)) + + Expect(netplan.Ethernets).To(HaveLen(1)) + Expect(netplan.Ethernets).To(HaveKey(ifName)) + + np := netplan.Ethernets[ifName] + Expect(np.Match.MacAddress).To(Equal(macAddr1Norm)) + Expect(np.SetName).To(Equal(ifName)) + Expect(np.Dhcp4).To(BeTrue()) + Expect(np.Dhcp6).To(BeTrue()) + Expect(np.MTU).To(BeEquivalentTo(9000)) + Expect(np.Nameservers.Addresses).To(Equal([]string{dnsServer1})) + Expect(np.Nameservers.Search).To(Equal([]string{searchDomain1})) + Expect(np.Routes).To(BeEmpty()) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/network.go b/pkg/vmprovider/providers/vsphere2/network/network.go new file mode 100644 index 000000000..e5d5d96e5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network.go @@ -0,0 +1,630 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//nolint:revive +package network + +import ( + goctx "context" + "fmt" + "net" + "strings" + "time" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + vimtypes "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type NetworkInterfaceResults struct { + Results []NetworkInterfaceResult +} + +type NetworkInterfaceResult struct { + IPConfigs []NetworkInterfaceIPConfig + MacAddress string + ExternalID string + NetworkID string + Backing object.NetworkReference + + Device vimtypes.BaseVirtualDevice + + // Fields from the InterfaceSpec used later during customization. + Name string + DHCP4 bool + DHCP6 bool + MTU int64 + Nameservers []string + SearchDomains []string + Routes []NetworkInterfaceRoute +} + +type NetworkInterfaceIPConfig struct { + IPCIDR string // IP address in CIDR notation e.g. 192.168.10.42/24 + IsIPv4 bool + Gateway string +} + +type NetworkInterfaceRoute struct { + To string + Via string + Metric int32 +} + +const ( + retryInterval = 100 * time.Millisecond + defaultEthernetCardType = "vmxnet3" +) + +var ( + // RetryTimeout is var so tests can change it to shorten tests until we get rid of the poll. + RetryTimeout = 15 * time.Second +) + +// CreateAndWaitForNetworkInterfaces creates the appropriate CRs for the VM's network +// interfaces, and then waits for them to be reconciled by NCP (NSX-T) or NetOP (VDS). +// +// Networking has always been kind of a pain and clunky for us, and unfortunately this +// code suffers gotchas and other not-so-great limitations. +// +// - Historically, this code used wait.PollImmediate() and we continue to do so here, +// but eventually we should Watch() these resources. Note though, that in the very +// common case, the CR is reconciled before our poll timeout, so that does save us +// from bailing out of the Reconcile. +// - Both NCP and NetOP CR Status inform us of the backing and IPAM info. However, for +// our InterfaceSpec we allow for DHCP but neither NCP nor NetOP has a way for us to +// mark the CR to don't do IPAM or to check DHCP is even enabled on the network. So +// this burns an IP, and the user must know that DHCP is actually configured. +// - CR naming has mostly been working by luck, and sometimes didn't offer very good +// discoverability. Here, with v1a2 we now have a "name" field in our InterfaceSpec, +// so we use that (BMV: need to double-check that field meets k8s name requirements) +// A longer term option is to use GenerateName to ensure a unique name, and then +// client.List() and filter by the OwnerRef to find the VM's network CRs, and to +// annotate the CRs to help identify which VM InterfaceSpec it corresponds to. +// Note that for existing v1a1 VMs we may need to add legacy name support here to +// find their interface CRs. +// - Instead of CreateOrUpdate, use CreateOrPatch to lessen the odds of blowing away +// any new fields. +func CreateAndWaitForNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + finder *find.Finder, + clusterMoRef *vimtypes.ManagedObjectReference, + interfaces []vmopv1.VirtualMachineNetworkInterfaceSpec) (NetworkInterfaceResults, error) { + + networkType := lib.GetNetworkProviderType() + if networkType == "" { + return NetworkInterfaceResults{}, fmt.Errorf("no network provider set") + } + + results := make([]NetworkInterfaceResult, 0, len(interfaces)) + + for i := range interfaces { + interfaceSpec := &interfaces[i] + + var result *NetworkInterfaceResult + var err error + + switch networkType { + case lib.NetworkProviderTypeVDS: + result, err = createNetOPNetworkInterface(vmCtx, client, vimClient, interfaceSpec) + case lib.NetworkProviderTypeNSXT: + result, err = createNCPNetworkInterface(vmCtx, client, vimClient, clusterMoRef, interfaceSpec) + case lib.NetworkProviderTypeNamed: + result, err = createNamedNetworkInterface(vmCtx, finder, interfaceSpec) + default: + err = fmt.Errorf("unsupported network provider envvar value: %q", networkType) + } + + if err != nil { + return NetworkInterfaceResults{}, + fmt.Errorf("network interface %q error: %w", interfaceSpec.Name, err) + } + + applyInterfaceSpecToResult(interfaceSpec, result) + results = append(results, *result) + } + + // TODO: Once we really support network changing on the fly, we need to keep track of now + // unused network interface CRDs so they can be deleted after they're removed from the VM + // via Reconfigure, instead of delaying that until the VM is deleted via GC. + + return NetworkInterfaceResults{ + Results: results, + }, nil +} + +// applyInterfaceSpecToResult applies the InterfaceSpec to results. Much of the InterfaceSpec - like DHCP - +// cannot be specified to the underlying network provider so apply those overrides to the results. +func applyInterfaceSpecToResult( + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec, + result *NetworkInterfaceResult) { + + // We don't really support IPv6 yet so don't enable it when the underlying provider didn't return any IPs. + dhcp4 := interfaceSpec.DHCP4 || len(result.IPConfigs) == 0 + dhcp6 := interfaceSpec.DHCP6 + + if len(interfaceSpec.Addresses) > 0 { + // The InterfaceSpec takes precedence over what underlying network provider says, so in this case it + // likely it did IPAM but we'll ignore those IPs. Providing static IPs via the Addresses field is + // probably not very common so override the IPConfigs so bootstrap has only one field to use. + result.IPConfigs = make([]NetworkInterfaceIPConfig, 0, len(interfaceSpec.Addresses)) + + for _, addr := range interfaceSpec.Addresses { + ip, _, err := net.ParseCIDR(addr) + if err != nil { + continue + } + + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: addr, + IsIPv4: ip.To4() != nil, + } + + if ipConfig.IsIPv4 { + dhcp4 = false + ipConfig.Gateway = interfaceSpec.Gateway4 + } else { + dhcp6 = false + ipConfig.Gateway = interfaceSpec.Gateway6 + } + + result.IPConfigs = append(result.IPConfigs, ipConfig) + } + } + + result.Name = interfaceSpec.Name + result.DHCP4 = dhcp4 + result.DHCP6 = dhcp6 + result.Nameservers = interfaceSpec.Nameservers + result.SearchDomains = interfaceSpec.SearchDomains + + if interfaceSpec.MTU != nil { + result.MTU = *interfaceSpec.MTU + } + for _, route := range interfaceSpec.Routes { + result.Routes = append(result.Routes, NetworkInterfaceRoute{To: route.To, Via: route.Via, Metric: route.Metric}) + } +} + +func createNamedNetworkInterface( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + if interfaceSpec.Network.Kind != "" || interfaceSpec.Network.APIVersion != "" { + return nil, fmt.Errorf("network TypeMeta not supported for name network: %v", interfaceSpec.Network.TypeMeta) + } + + networkName := interfaceSpec.Network.Name + if networkName == "" { + return nil, fmt.Errorf("network name is required") + } + + backing, err := finder.Network(vmCtx, networkName) + if err != nil { + return nil, fmt.Errorf("unable to find named network %q: %w", networkName, err) + } + + return &NetworkInterfaceResult{ + NetworkID: networkName, + Backing: backing, + }, nil +} + +// NetOPCRName returns the name to be used for the NetOP NetworkInterface CR. +func NetOPCRName(vmName, networkName, interfaceName string, isV1A1 bool) string { + var name string + + if isV1A1 { + // Old naming convention: each network can really only have 1 NIC. + if networkName != "" { + name = fmt.Sprintf("%s-%s", networkName, vmName) + } else { + name = vmName + } + } else { + if networkName != "" { + name = fmt.Sprintf("%s-%s-%s", vmName, networkName, interfaceName) + } else { + name = fmt.Sprintf("%s-%s", vmName, interfaceName) + } + } + + return name +} + +func createNetOPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + if kind := interfaceSpec.Network.Kind; kind != "" && kind != "Network" { + return nil, fmt.Errorf("network kind %q is not supported for VDS", kind) + } + + // If empty, NetOP will try to select a namespace default. + networkName := interfaceSpec.Network.Name + + netIf := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: NetOPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), + Namespace: vmCtx.VM.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(vmCtx, client, netIf, func() error { + if err := controllerutil.SetOwnerReference(vmCtx.VM, netIf, client.Scheme()); err != nil { + // If this fails we likely have an object name collision, and we're in a tough spot. + return err + } + + netIf.Spec.NetworkName = networkName + // NetOP only defines a VMXNet3 type, but it doesn't really matter for our purposes. + netIf.Spec.Type = netopv1alpha1.NetworkInterfaceTypeVMXNet3 + return nil + }) + + if err != nil { + return nil, err + } + + netIf, err = waitForReadyNetworkInterface(vmCtx, client, netIf.Name) + if err != nil { + return nil, err + } + + return netOpNetIfToResult(vimClient, netIf), nil +} + +func netOpNetIfToResult( + vimClient *vim25.Client, + netIf *netopv1alpha1.NetworkInterface) *NetworkInterfaceResult { + + ipConfigs := make([]NetworkInterfaceIPConfig, 0, len(netIf.Status.IPConfigs)) + for _, ip := range netIf.Status.IPConfigs { + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: ipCIDRNotation(ip.IP, ip.SubnetMask, ip.IPFamily == netopv1alpha1.IPv4Protocol), + IsIPv4: ip.IPFamily == netopv1alpha1.IPv4Protocol, + Gateway: ip.Gateway, + } + ipConfigs = append(ipConfigs, ipConfig) + } + + pgObjRef := vimtypes.ManagedObjectReference{ + Type: "DistributedVirtualPortgroup", + Value: netIf.Status.NetworkID, + } + + return &NetworkInterfaceResult{ + IPConfigs: ipConfigs, + MacAddress: netIf.Status.MacAddress, // Not set by NetOP. + ExternalID: netIf.Status.ExternalID, // Ditto. + NetworkID: netIf.Status.NetworkID, + Backing: object.NewDistributedVirtualPortgroup(vimClient, pgObjRef), + } +} + +func findNetOPCondition( + netIf *netopv1alpha1.NetworkInterface, + condType netopv1alpha1.NetworkInterfaceConditionType) *netopv1alpha1.NetworkInterfaceCondition { + + for i := range netIf.Status.Conditions { + if netIf.Status.Conditions[i].Type == condType { + return &netIf.Status.Conditions[i] + } + } + return nil +} + +func waitForReadyNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + name string) (*netopv1alpha1.NetworkInterface, error) { + + netIf := &netopv1alpha1.NetworkInterface{} + netIfKey := types.NamespacedName{Namespace: vmCtx.VM.Namespace, Name: name} + + // TODO: Watch() this type instead. + err := wait.PollUntilContextTimeout(vmCtx, retryInterval, RetryTimeout, true, func(_ goctx.Context) (bool, error) { + if err := client.Get(vmCtx, netIfKey, netIf); err != nil { + return false, ctrlruntime.IgnoreNotFound(err) + } + + cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceReady) + return cond != nil && cond.Status == corev1.ConditionTrue, nil + }) + + if err != nil { + if wait.Interrupted(err) { + // Try to return a more meaningful error when timed out. + if cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceFailure); cond != nil && cond.Status == corev1.ConditionTrue { + return nil, fmt.Errorf("network interface failure: %s - %s", cond.Reason, cond.Message) + } + if cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceReady); cond != nil && cond.Status == corev1.ConditionFalse { + return nil, fmt.Errorf("network interface is not ready: %s - %s", cond.Reason, cond.Message) + } + return nil, fmt.Errorf("network interface is not ready yet") + } + + return nil, err + } + + return netIf, nil +} + +// NCPCRName returns the name to be used for the NCP VirtualNetworkInterface CR. +func NCPCRName(vmName, networkName, interfaceName string, isV1A1 bool) string { + var name string + + if isV1A1 { + name = fmt.Sprintf("%s-lsp", vmName) + if networkName != "" { + name = fmt.Sprintf("%s-%s", networkName, name) + } + + } else { + if networkName != "" { + name = fmt.Sprintf("%s-%s-%s", vmName, networkName, interfaceName) + } else { + name = fmt.Sprintf("%s-%s", vmName, interfaceName) + } + } + + return name +} + +func createNCPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + clusterMoRef *vimtypes.ManagedObjectReference, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + // TODO: Do we need to still support the odd-ball NetOP in NSX-T? Sigh. Do that check here if needed. + if kind := interfaceSpec.Network.Kind; kind != "" && kind != "VirtualNetwork" { + return nil, fmt.Errorf("network kind %q is not supported for NCP", kind) + } + + // If empty, NCP will use the namespace default. + networkName := interfaceSpec.Network.Name + + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: NCPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), + Namespace: vmCtx.VM.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(vmCtx, client, vnetIf, func() error { + if err := controllerutil.SetOwnerReference(vmCtx.VM, vnetIf, client.Scheme()); err != nil { + return err + } + + vnetIf.Spec.VirtualNetwork = networkName + return nil + }) + + if err != nil { + return nil, err + } + + vnetIf, err = waitForReadyNCPNetworkInterface(vmCtx, client, vnetIf.Name) + if err != nil { + return nil, err + } + + return ncpNetIfToResult(vmCtx, vimClient, clusterMoRef, vnetIf) +} + +func ncpNetIfToResult( + ctx goctx.Context, + vimClient *vim25.Client, + clusterMoRef *vimtypes.ManagedObjectReference, + vnetIf *ncpv1alpha1.VirtualNetworkInterface) (*NetworkInterfaceResult, error) { + + // NSX-T makes the backing determination difficult: NsxLogicalSwitchID must be mapped to an + // actual DVPG since that is the backing, but the DVPG can, in some very rare but supported + // configurations, vary between CCRs. If we know the CCR - ether the VM already exists, or + // (for later work) we might pre-determine CCR w/o placement if there is only one possibility - + // get that backing now. + // Otherwise, we'll do it post-placement via ResolveNCPBackingPostPlacement() so that we create + // the VM with the correct backing. That means we cannot make this a part of the PlaceVMxCluster() + // ConfigSpec since we don't know the backing: we'd have to pre-filter the placement candidates. + // What a mess. This is an unfortunate decision that forces mapping logic to every NCP consumer. + + var backing object.NetworkReference + networkID := vnetIf.Status.ProviderStatus.NsxLogicalSwitchID + + if clusterMoRef != nil { + ccr := object.NewClusterComputeResource(vimClient, *clusterMoRef) + + networkRef, err := searchNsxtNetworkReference(ctx, ccr, networkID) + if err != nil { + return nil, err + } + + backing = networkRef + } + + var ipConfigs []NetworkInterfaceIPConfig + if ipAddress := vnetIf.Status.IPAddresses; len(ipAddress) == 0 || (len(ipAddress) == 1 && ipAddress[0].IP == "") { + // NCP's way of saying DHCP. + } else { + // Historically, we only grabbed the first entry and assume it is always IPv4 (!!!). Try to do slightly better. + for _, ipAddr := range ipAddress { + if ipAddr.IP == "" { + continue + } + + isIPv4 := net.ParseIP(ipAddr.IP).To4() != nil + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: ipCIDRNotation(ipAddr.IP, ipAddr.SubnetMask, isIPv4), + IsIPv4: isIPv4, + Gateway: ipAddr.Gateway, + } + + ipConfigs = append(ipConfigs, ipConfig) + } + } + + result := &NetworkInterfaceResult{ + IPConfigs: ipConfigs, + MacAddress: vnetIf.Status.MacAddress, + ExternalID: vnetIf.Status.InterfaceID, + NetworkID: networkID, + Backing: backing, + } + + return result, nil +} + +func waitForReadyNCPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + name string) (*ncpv1alpha1.VirtualNetworkInterface, error) { + + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{} + vnetIfKey := types.NamespacedName{Namespace: vmCtx.VM.Namespace, Name: name} + + // TODO: Watch() this type instead. + err := wait.PollUntilContextTimeout(vmCtx, retryInterval, RetryTimeout, true, func(_ goctx.Context) (bool, error) { + if err := client.Get(vmCtx, vnetIfKey, vnetIf); err != nil { + return false, ctrlruntime.IgnoreNotFound(err) + } + + for _, condition := range vnetIf.Status.Conditions { + // TODO: Does NCP define condition constants? + if strings.Contains(condition.Type, "Ready") && strings.Contains(condition.Status, "True") { + return true, nil + } + } + + return false, nil + }) + + if err != nil { + if wait.Interrupted(err) { + // Try to return a more meaningful error when timed out. + for _, cond := range vnetIf.Status.Conditions { + if strings.Contains(cond.Type, "Ready") && !strings.Contains(cond.Status, "True") { + return nil, fmt.Errorf("network interface is not ready: %s - %s", cond.Reason, cond.Message) + } + } + // TODO: NCP also has an annotation but that usually doesn't provide very useful details. + return nil, fmt.Errorf("network interface is not ready yet") + } + + return nil, err + } + + if vnetIf.Status.ProviderStatus == nil { + return nil, fmt.Errorf("network interface is ready but does not have provider status") + } + + return vnetIf, nil +} + +// ipCIDRNotation takes the IP and subnet mask and returns the IP in CIDR notation. +// TODO: Better error checking. Nail down exactly how we want handle IPv4inV6 addresses. +func ipCIDRNotation(ip string, mask string, isIPv4 bool) string { + if isIPv4 { + ipNet := net.IPNet{ + IP: net.ParseIP(ip).To4(), + Mask: net.IPMask(net.ParseIP(mask).To4()), + } + return ipNet.String() + } + + ipNet := net.IPNet{ + IP: net.ParseIP(ip).To16(), + Mask: net.IPMask(net.ParseIP(mask).To16()), + } + + return ipNet.String() +} + +// CreateDefaultEthCard creates a default Ethernet card attached to the backing. This is used +// when the VM Class ConfigSpec does not have a device entry for a VM Spec network interface, +// so we need a new device. +func CreateDefaultEthCard( + ctx goctx.Context, + result *NetworkInterfaceResult) (vimtypes.BaseVirtualDevice, error) { + + // We may not have the backing yet if this is NSX-T. The backing will be resolved after placement + // when we'll know the CCR, so we can resolve the correct DVPG. + if result.Backing == nil { + return nil, nil + } + + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + if err != nil { + return nil, fmt.Errorf("unable to get ethernet card backing info for network %v: %w", result.Backing.Reference(), err) + } + + dev, err := object.EthernetCardTypes().CreateEthernetCard(defaultEthernetCardType, backing) + if err != nil { + return nil, fmt.Errorf("unable to create ethernet card network %v: %w", result.Backing.Reference(), err) + } + + ethCard := dev.(vimtypes.BaseVirtualEthernetCard).GetVirtualEthernetCard() + ethCard.ExternalId = result.ExternalID + if result.MacAddress != "" { + ethCard.MacAddress = result.MacAddress + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeManual) + } else { + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeGenerated) // TODO: Or TypeAssigned? + } + + return dev, nil +} + +// ApplyInterfaceResultToVirtualEthCard applies the interface result from the NetOP/NCP +// provider to an existing Ethernet device from the class ConfigSpec. +func ApplyInterfaceResultToVirtualEthCard( + ctx goctx.Context, + ethCard *vimtypes.VirtualEthernetCard, + result *NetworkInterfaceResult) error { + + ethCard.ExternalId = result.ExternalID + if result.MacAddress != "" { + // BMV: Too much confusion and possible breakage if we don't honor the provider MAC. + // Otherwise, IMO a foot gun and will break on setups that enforce MAC filtering. + ethCard.MacAddress = result.MacAddress + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeManual) + } else { //nolint + // BMV: IMO this must be Generated/TypeAssigned to avoid major foot gun, but we have tests assuming + // this is left as-is. + // We should have a MAC address field to the VM.Spec if we want this to be specified by the user. + // ethCard.MacAddress = "" + // ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeGenerated) + } + + // We may not have the backing yet if this is NSX-T. The backing will be resolved after placement + // when we'll know the CCR, so we can resolve the correct DVPG. + if result.Backing != nil { + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + if err != nil { + return fmt.Errorf("unable to get ethernet card backing info for network %v: %w", result.NetworkID, err) + } + ethCard.Backing = backing + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go b/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go new file mode 100644 index 000000000..29fcd237c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestNetwork(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Network Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/network/network_test.go b/pkg/vmprovider/providers/vsphere2/network/network_test.go new file mode 100644 index 000000000..c57a5e1ab --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network_test.go @@ -0,0 +1,344 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + goctx "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + interfaceSpecs []vmopv1.VirtualMachineNetworkInterfaceSpec + + results network.NetworkInterfaceResults + err error + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "network-test-vm", + Namespace: "network-test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithName("network_test"), + VM: vm, + } + + interfaceSpecs = nil + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Context("Named Network", func() { + // Use network vcsim automatically creates. + const networkName = "DC0_DVPG0" + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + }) + + Context("network exists", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: networkName}, + DHCP6: true, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(1)) + + result := results.Results[0] + By("has expected backing", func() { + Expect(result.Backing).ToNot(BeNil()) + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + Expect(err).ToNot(HaveOccurred()) + backingInfo, ok := backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).To(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + + Expect(result.DHCP4).To(BeTrue()) + Expect(result.DHCP6).To(BeTrue()) // Only enabled if explicitly requested (which it is above). + }) + }) + + Context("network does not exist", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: "bogus"}, + }, + } + }) + + It("returns error", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unable to find named network")) + Expect(results.Results).To(BeEmpty()) + }) + }) + }) + + Context("VDS", func() { + const ( + interfaceName = "eth0" + networkName = "my-vds-network" + ) + + BeforeEach(func() { + network.RetryTimeout = 1 * time.Second + testConfig.WithNetworkEnv = builder.NetworkEnvVDS + }) + + Context("Simulate workflow", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + } + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NetOP reconcile", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.NetworkName).To(Equal(networkName)) + + netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value + netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.110", + IPFamily: netopv1alpha1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + IPFamily: netopv1alpha1.IPv6Protocol, + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(BeEmpty()) + Expect(result.ExternalID).To(BeEmpty()) + Expect(result.NetworkID).To(Equal(ctx.NetworkRef.Reference().Value)) + Expect(result.Backing).ToNot(BeNil()) + Expect(result.Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + }) + }) + }) + + Context("NCP", func() { + const ( + interfaceName = "eth0" + interfaceID = "my-interface-id" + networkName = "my-ncp-network" + macAddress = "01-23-45-67-89-AB-CD-EF" + ) + + BeforeEach(func() { + network.RetryTimeout = 1 * time.Second + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + }) + + Context("Simulate workflow", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + } + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NCP reconcile", func() { + netInterface := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.VirtualNetwork).To(Equal(networkName)) + + netInterface.Status.InterfaceID = interfaceID + netInterface.Status.MacAddress = macAddress + netInterface.Status.ProviderStatus = &ncpv1alpha1.VirtualNetworkInterfaceProviderStatus{ + NsxLogicalSwitchID: builder.NsxTLogicalSwitchUUID, + } + netInterface.Status.IPAddresses = []ncpv1alpha1.VirtualNetworkInterfaceIP{ + { + IP: "192.168.1.110", + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []ncpv1alpha1.VirtualNetworkCondition{ + { + Type: "Ready", + Status: "True", + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(Equal(macAddress)) + Expect(result.ExternalID).To(Equal(interfaceID)) + Expect(result.NetworkID).To(Equal(builder.NsxTLogicalSwitchUUID)) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + + // Without the ClusterMoRef on the first call this will be nil for NSXT. + Expect(result.Backing).To(BeNil()) + + clusterMoRef := ctx.GetSingleClusterCompute().Reference() + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + &clusterMoRef, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(1)) + Expect(results.Results[0].Backing).ToNot(BeNil()) + Expect(results.Results[0].Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/nsxt.go b/pkg/vmprovider/providers/vsphere2/network/nsxt.go new file mode 100644 index 000000000..d08c6a6b1 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/nsxt.go @@ -0,0 +1,117 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + goctx "context" + "fmt" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +// ResolveBackingPostPlacement fixes up the backings where we did not know the CCR until after +// placement. This should be called if CreateAndWaitForNetworkInterfaces() was called with a nil +// clusterMoRef. Returns true if a backing was resolved, so the ConfigSpec needs to be updated. +func ResolveBackingPostPlacement( + ctx goctx.Context, + vimClient *vim25.Client, + clusterMoRef vimtypes.ManagedObjectReference, + results *NetworkInterfaceResults) (bool, error) { + + if len(results.Results) == 0 { + return false, nil + } + + networkType := lib.GetNetworkProviderType() + if networkType == "" { + return false, fmt.Errorf("no network provider set") + } + + ccr := object.NewClusterComputeResource(vimClient, clusterMoRef) + fixedUp := false + + for idx := range results.Results { + if results.Results[idx].Backing != nil { + continue + } + + var backing object.NetworkReference + var err error + + switch networkType { + case lib.NetworkProviderTypeNSXT: + backing, err = searchNsxtNetworkReference(ctx, ccr, results.Results[idx].NetworkID) + if err != nil { + err = fmt.Errorf("post placement NSX-T backing fixup failed: %w", err) + } + default: + err = fmt.Errorf("only NSX-T networks are expected to need post placement backing fixup") + } + + if err != nil { + return false, err + } + + fixedUp = true + results.Results[idx].Backing = backing + } + + return fixedUp, nil +} + +// searchNsxtNetworkReference takes in NSX-T LogicalSwitchUUID and returns the reference of the network. +func searchNsxtNetworkReference( + ctx goctx.Context, + ccr *object.ClusterComputeResource, + networkID string) (object.NetworkReference, error) { + + // This is more or less how the old code did it. We could save repeated work by moving this + // into the callers since it will always be for the same CCR, but the common case is one NIC, + // or at most a handful, so that's for later. + var obj mo.ClusterComputeResource + if err := ccr.Properties(ctx, ccr.Reference(), []string{"network"}, &obj); err != nil { + return nil, err + } + + var dvpgsMoRefs []vimtypes.ManagedObjectReference + for _, n := range obj.Network { + if n.Type == "DistributedVirtualPortgroup" { + dvpgsMoRefs = append(dvpgsMoRefs, n.Reference()) + } + } + + if len(dvpgsMoRefs) == 0 { + return nil, fmt.Errorf("ClusterComputeResource %s has no DVPGs", ccr.Reference().Value) + } + + var dvpgs []mo.DistributedVirtualPortgroup + err := property.DefaultCollector(ccr.Client()).Retrieve(ctx, dvpgsMoRefs, []string{"config.logicalSwitchUuid"}, &dvpgs) + if err != nil { + return nil, err + } + + var dvpgMoRefs []vimtypes.ManagedObjectReference + for _, dvpg := range dvpgs { + if dvpg.Config.LogicalSwitchUuid == networkID { + dvpgMoRefs = append(dvpgMoRefs, dvpg.Reference()) + } + } + + switch len(dvpgMoRefs) { + case 1: + return object.NewDistributedVirtualPortgroup(ccr.Client(), dvpgMoRefs[0]), nil + case 0: + return nil, fmt.Errorf("no DVPG with NSX-T network ID %q found", networkID) + default: + // The LogicalSwitchUuid is supposed to be unique per CCR, so this is likely an NCP + // misconfiguration, and we don't know which one to pick. + return nil, fmt.Errorf("multiple DVPGs (%d) with NSX-T network ID %q found", len(dvpgMoRefs), networkID) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go b/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go new file mode 100644 index 000000000..35925553a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("ResolveBackingPostPlacement", func() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + + results *network.NetworkInterfaceResults + fixedUp bool + err error + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + fixedUp, err = network.ResolveBackingPostPlacement( + ctx, + ctx.VCClient.Client, + ctx.GetSingleClusterCompute().Reference(), + results) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Context("returns success", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + + results = &network.NetworkInterfaceResults{ + Results: []network.NetworkInterfaceResult{ + { + NetworkID: builder.NsxTLogicalSwitchUUID, + Backing: nil, + }, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(fixedUp).To(BeTrue()) + + Expect(results.Results).To(HaveLen(1)) + By("should populate the backing", func() { + backing := results.Results[0].Backing + Expect(backing).ToNot(BeNil()) + Expect(backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go new file mode 100644 index 000000000..6a28e8a12 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go @@ -0,0 +1,206 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + goctx "context" + "fmt" + "strings" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +// Recommendation is the info about a placement recommendation. +type Recommendation struct { + PoolMoRef types.ManagedObjectReference + HostMoRef *types.ManagedObjectReference + // TODO: Datastore, whatever else as we need it. +} + +func relocateSpecToRecommendation(relocateSpec *types.VirtualMachineRelocateSpec) *Recommendation { + // Instance Storage requires the host. + if relocateSpec == nil || relocateSpec.Pool == nil || relocateSpec.Host == nil { + return nil + } + + return &Recommendation{ + PoolMoRef: *relocateSpec.Pool, + HostMoRef: relocateSpec.Host, + } +} + +func clusterPlacementActionToRecommendation(action types.ClusterClusterInitialPlacementAction) *Recommendation { + return &Recommendation{ + PoolMoRef: action.Pool, + HostMoRef: action.TargetHost, + } +} + +func CheckPlacementRelocateSpec(spec *types.VirtualMachineRelocateSpec) error { + if spec == nil { + return fmt.Errorf("RelocateSpec is nil") + } + if spec.Host == nil { + return fmt.Errorf("RelocateSpec does not have a host") + } + if spec.Pool == nil { + return fmt.Errorf("RelocateSpec does not have a resource pool") + } + if spec.Datastore == nil { + return fmt.Errorf("RelocateSpec does not have a datastore") + } + return nil +} + +func ParseRelocateVMResponse( + vmCtx context.VirtualMachineContextA2, + res *types.PlacementResult) *types.VirtualMachineRelocateSpec { + + for _, r := range res.Recommendations { + if r.Reason == string(types.RecommendationReasonCodeXvmotionPlacement) { + for _, a := range r.Action { + if pa, ok := a.(*types.PlacementAction); ok { + if err := CheckPlacementRelocateSpec(pa.RelocateSpec); err != nil { + vmCtx.Logger.V(6).Info("Skipped RelocateSpec", + "reason", err.Error(), "relocateSpec", pa.RelocateSpec) + continue + } + + return pa.RelocateSpec + } + } + } + } + + return nil +} + +func CloneVMRelocateSpec( + vmCtx context.VirtualMachineContextA2, + cluster *object.ClusterComputeResource, + vmRef types.ManagedObjectReference, + cloneSpec *types.VirtualMachineCloneSpec) (*types.VirtualMachineRelocateSpec, error) { + + placementSpec := types.PlacementSpec{ + PlacementType: string(types.PlacementSpecPlacementTypeClone), + CloneSpec: cloneSpec, + RelocateSpec: &cloneSpec.Location, + CloneName: cloneSpec.Config.Name, + Vm: &vmRef, + } + + resp, err := cluster.PlaceVm(vmCtx, placementSpec) + if err != nil { + return nil, err + } + + rSpec := ParseRelocateVMResponse(vmCtx, resp) + if rSpec == nil { + return nil, fmt.Errorf("no valid placement action") + } + + return rSpec, nil +} + +// PlaceVMForCreate determines the suitable placement candidates in the cluster. +func PlaceVMForCreate( + ctx goctx.Context, + cluster *object.ClusterComputeResource, + configSpec *types.VirtualMachineConfigSpec) ([]Recommendation, error) { + + placementSpec := types.PlacementSpec{ + PlacementType: string(types.PlacementSpecPlacementTypeCreate), + ConfigSpec: configSpec, + } + + resp, err := cluster.PlaceVm(ctx, placementSpec) + if err != nil { + return nil, err + } + + var recommendations []Recommendation + + for _, r := range resp.Recommendations { + if r.Reason != string(types.RecommendationReasonCodeXvmotionPlacement) { + continue + } + + for _, a := range r.Action { + if pa, ok := a.(*types.PlacementAction); ok { + if r := relocateSpecToRecommendation(pa.RelocateSpec); r != nil { + recommendations = append(recommendations, *r) + } + } + } + } + + return recommendations, nil +} + +// ClusterPlaceVMForCreate determines the suitable cluster placement among the specified ResourcePools. +func ClusterPlaceVMForCreate( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + resourcePoolsMoRefs []types.ManagedObjectReference, + configSpec *types.VirtualMachineConfigSpec, + needsHost bool) ([]Recommendation, error) { + + // Work around PlaceVmsXCluster bug that crashes vpxd when ConfigSpec.Files is nil. + cs := *configSpec + cs.Files = new(types.VirtualMachineFileInfo) + + placementSpec := types.PlaceVmsXClusterSpec{ + ResourcePools: resourcePoolsMoRefs, + VmPlacementSpecs: []types.PlaceVmsXClusterSpecVmPlacementSpec{ + { + ConfigSpec: cs, + }, + }, + HostRecommRequired: &needsHost, + } + + vmCtx.Logger.V(6).Info("PlaceVmxCluster request", "placementSpec", placementSpec) + + resp, err := object.NewRootFolder(vcClient).PlaceVmsXCluster(vmCtx, placementSpec) + if err != nil { + return nil, err + } + + vmCtx.Logger.V(6).Info("PlaceVmxCluster response", "resp", resp) + + if len(resp.Faults) != 0 { + var faultMgs []string + for _, f := range resp.Faults { + msgs := make([]string, 0, len(f.Faults)) + for _, ff := range f.Faults { + msgs = append(msgs, ff.LocalizedMessage) + } + faultMgs = append(faultMgs, + fmt.Sprintf("ResourcePool %s faults: %s", f.ResourcePool.Value, strings.Join(msgs, ", "))) + } + return nil, fmt.Errorf("PlaceVmsXCluster faults: %v", faultMgs) + } + + var recommendations []Recommendation + + for _, info := range resp.PlacementInfos { + if info.Recommendation.Reason != string(types.RecommendationReasonCodeXClusterPlacement) { + continue + } + + for _, a := range info.Recommendation.Action { + if ca, ok := a.(*types.ClusterClusterInitialPlacementAction); ok { + if r := clusterPlacementActionToRecommendation(*ca); r != nil { + recommendations = append(recommendations, *r) + } + } + } + } + + return recommendations, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go new file mode 100644 index 000000000..8dc78a2a8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go @@ -0,0 +1,136 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/placement" +) + +func createRelocateSpec() *types.VirtualMachineRelocateSpec { + spec := &types.VirtualMachineRelocateSpec{} + spec.Host = &types.ManagedObjectReference{} + spec.Pool = &types.ManagedObjectReference{} + spec.Datastore = &types.ManagedObjectReference{} + return spec +} + +func createValidPlacementAction() (types.BaseClusterAction, *types.VirtualMachineRelocateSpec) { + action := types.PlacementAction{} + action.RelocateSpec = createRelocateSpec() + return types.BaseClusterAction(&action), action.RelocateSpec +} + +func createInvalidPlacementAction() types.BaseClusterAction { + action := types.PlacementAction{} + action.RelocateSpec = createRelocateSpec() + action.RelocateSpec.Host = nil + return types.BaseClusterAction(&action) +} + +func createStoragePlacementAction() types.BaseClusterAction { + action := types.StoragePlacementAction{} + action.RelocateSpec = *createRelocateSpec() + return types.BaseClusterAction(&action) +} + +func createInvalidRecommendation() types.ClusterRecommendation { + r := types.ClusterRecommendation{} + r.Reason = string(types.RecommendationReasonCodeXvmotionPlacement) + r.Action = append(r.Action, createStoragePlacementAction()) + r.Action = append(r.Action, createInvalidPlacementAction()) + return r +} + +func createValidRecommendation() (types.ClusterRecommendation, *types.VirtualMachineRelocateSpec) { + r := createInvalidRecommendation() + a, s := createValidPlacementAction() + r.Action = append(r.Action, a) + return r, s +} + +var _ = Describe("ParsePlaceVMResponse", func() { + + Context("when response is valid", func() { + Specify("PlaceVm Response is valid", func() { + res := types.PlacementResult{} + res.Recommendations = append(res.Recommendations, createInvalidRecommendation(), createInvalidRecommendation()) + rec, _ := createValidRecommendation() + rec.Reason = string(types.RecommendationReasonCodePowerOnVm) + res.Recommendations = append(res.Recommendations, rec) + rec, spec := createValidRecommendation() + res.Recommendations = append(res.Recommendations, rec) + + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).NotTo(BeNil()) + Expect(rSpec.Host).To(BeEquivalentTo(spec.Host)) + Expect(rSpec.Pool).To(BeEquivalentTo(spec.Pool)) + Expect(rSpec.Datastore).To(BeEquivalentTo(spec.Datastore)) + }) + }) + + Context("when response is not valid", func() { + Specify("PlaceVm Response without recommendations", func() { + res := types.PlacementResult{} + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).To(BeNil()) + }) + }) + + Context("when response is not valid", func() { + Specify("PlaceVm Response with invalid recommendations only", func() { + res := types.PlacementResult{} + res.Recommendations = append(res.Recommendations, createInvalidRecommendation(), createInvalidRecommendation()) + rec, _ := createValidRecommendation() + rec.Reason = string(types.RecommendationReasonCodePowerOnVm) + res.Recommendations = append(res.Recommendations, rec) + + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).To(BeNil()) + }) + }) +}) + +var _ = Describe("CheckPlacementRelocateSpec", func() { + + Context("when relocation spec is valid", func() { + Specify("Relocation spec is valid", func() { + spec := createRelocateSpec() + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeTrue()) + }) + }) + + Context("when relocation spec is not valid", func() { + Specify("Relocation spec is nil", func() { + isValid := placement.CheckPlacementRelocateSpec(nil) + Expect(isValid).To(BeFalse()) + }) + + Specify("Host is nil", func() { + spec := createRelocateSpec() + spec.Host = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + + Specify("Pool is nil", func() { + spec := createRelocateSpec() + spec.Pool = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + + Specify("Datastore is nil", func() { + spec := createRelocateSpec() + spec.Datastore = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go b/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go new file mode 100644 index 000000000..377f4133c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("Placement", vcSimPlacement) +} + +var suite = builder.NewTestSuite() + +func TestPlacement(t *testing.T) { + suite.Register(t, "VMProvider Placement", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go b/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go new file mode 100644 index 000000000..09717487e --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go @@ -0,0 +1,333 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + goctx "context" + "fmt" + "math/rand" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +type Result struct { + ZonePlacement bool + InstanceStoragePlacement bool + ZoneName string + HostMoRef *types.ManagedObjectReference + PoolMoRef types.ManagedObjectReference + // TODO: Datastore, whatever else as we need it. +} + +func doesVMNeedPlacement(vmCtx context.VirtualMachineContextA2) (res Result, needZonePlacement, needInstanceStoragePlacement bool) { + if lib.IsWcpFaultDomainsFSSEnabled() { + res.ZonePlacement = true + + if zoneName := vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey]; zoneName != "" { + // Zone has already been selected. + res.ZoneName = zoneName + } else { + // VM does not have a zone already assigned so we need to select one. + needZonePlacement = true + } + } + + if lib.IsInstanceStorageFSSEnabled() { + if instancestorage.IsPresent(vmCtx.VM) { + res.InstanceStoragePlacement = true + + if hostMoID := vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey]; hostMoID != "" { + // Host has already been selected. + res.HostMoRef = &types.ManagedObjectReference{Type: "HostSystem", Value: hostMoID} + } else { + // VM has InstanceStorage volumes so we need to select a host. + needInstanceStoragePlacement = true + } + } + } + + return +} + +// lookupChildRPs lookups the child ResourcePool under each parent ResourcePool. A VM with a ResourcePolicy +// may specify a child ResourcePool that the VM will be created under. +func lookupChildRPs( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + rpMoIDs []string, + zoneName, childRPName string) []string { + + childRPMoIDs := make([]string, 0, len(rpMoIDs)) + + for _, rpMoID := range rpMoIDs { + rp := object.NewResourcePool(vcClient, types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + childRP, err := vcenter.GetChildResourcePool(vmCtx, rp, childRPName) + if err != nil { + vmCtx.Logger.Error(err, "Skipping this resource pool since failed to get child ResourcePool", + "zone", zoneName, "parentRPMoID", rpMoID, "childRPName", childRPName) + continue + } + + childRPMoIDs = append(childRPMoIDs, childRP.Reference().Value) + } + + return childRPMoIDs +} + +// getPlacementCandidates determines the candidate resource pools for VM placement. +func getPlacementCandidates( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + vcClient *vim25.Client, + zonePlacement bool, + childRPName string) (map[string][]string, error) { + + var zones []topologyv1.AvailabilityZone + + if zonePlacement { + z, err := topology.GetAvailabilityZones(vmCtx, client) + if err != nil { + return nil, err + } + + zones = z + } else { + // Consider candidates only within the already assigned zone. + // NOTE: GetAvailabilityZone() will return a "default" AZ when the FSS is not enabled. + zone, err := topology.GetAvailabilityZone(vmCtx, client, vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey]) + if err != nil { + return nil, err + } + + zones = append(zones, zone) + } + + candidates := map[string][]string{} + + for _, zone := range zones { + nsInfo, ok := zone.Spec.Namespaces[vmCtx.VM.Namespace] + if !ok { + continue + } + + var rpMoIDs []string + if len(nsInfo.PoolMoIDs) != 0 { + rpMoIDs = nsInfo.PoolMoIDs + } else { + rpMoIDs = []string{nsInfo.PoolMoId} + } + + if childRPName != "" { + childRPMoIDs := lookupChildRPs(vmCtx, vcClient, rpMoIDs, zone.Name, childRPName) + if len(childRPMoIDs) == 0 { + vmCtx.Logger.Info("Zone had no candidates after looking up children ResourcePools", + "zone", zone.Name, "rpMoIDs", rpMoIDs, "childRPName", childRPName) + continue + } + rpMoIDs = childRPMoIDs + } + + candidates[zone.Name] = rpMoIDs + } + + return candidates, nil +} + +func rpMoIDToCluster( + ctx goctx.Context, + vcClient *vim25.Client, + rpMoRef types.ManagedObjectReference) (*object.ClusterComputeResource, error) { + + cluster, err := object.NewResourcePool(vcClient, rpMoRef).Owner(ctx) + if err != nil { + return nil, err + } + + return object.NewClusterComputeResource(vcClient, cluster.Reference()), nil +} + +// getPlacementRecommendations calls DRS PlaceVM to determine clusters suitable for placement. +func getPlacementRecommendations( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + candidates map[string][]string, + configSpec *types.VirtualMachineConfigSpec) map[string][]Recommendation { + + recommendations := map[string][]Recommendation{} + + for zoneName, rpMoIDs := range candidates { + for _, rpMoID := range rpMoIDs { + rpMoRef := types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID} + + cluster, err := rpMoIDToCluster(vmCtx, vcClient, rpMoRef) + if err != nil { + vmCtx.Logger.Error(err, "failed to get CCR from RP", "zone", zoneName, "rpMoID", rpMoID) + continue + } + + recs, err := PlaceVMForCreate(vmCtx, cluster, configSpec) + if err != nil { + vmCtx.Logger.Error(err, "PlaceVM failed", "zone", zoneName, + "clusterMoID", cluster.Reference().Value, "rpMoID", rpMoID) + continue + } + + if len(recs) == 0 { + vmCtx.Logger.Info("No placement recommendations", "zone", zoneName, + "clusterMoID", cluster.Reference().Value, "rpMoID", rpMoID) + continue + } + + // Replace the resource pool returned by PlaceVM - that is the cluster's root RP - with + // our more specific namespace or namespace child RP since this VM needs to be under the + // more specific RP. This makes the recommendations returned here the same as what zonal + // would return. + for idx := range recs { + recs[idx].PoolMoRef = rpMoRef + } + + recommendations[zoneName] = append(recommendations[zoneName], recs...) + } + } + + vmCtx.Logger.V(5).Info("Placement recommendations", "recommendations", recommendations) + + return recommendations +} + +// getZonalPlacementRecommendations calls DRS PlaceVmsXCluster to determine clusters suitable for placement. +func getZonalPlacementRecommendations( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + candidates map[string][]string, + configSpec *types.VirtualMachineConfigSpec, + needsHost bool) map[string][]Recommendation { + + rpMOToZone := map[types.ManagedObjectReference]string{} + var candidateRPMoRefs []types.ManagedObjectReference + + for zoneName, rpMoIDs := range candidates { + for _, rpMoID := range rpMoIDs { + rpMoRef := types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID} + candidateRPMoRefs = append(candidateRPMoRefs, rpMoRef) + rpMOToZone[rpMoRef] = zoneName + } + } + + var recs []Recommendation + + if len(candidateRPMoRefs) == 1 { + // If there is only one candidate, we might be able to skip some work. + + if needsHost { + // This is a hack until PlaceVmsXCluster() supports instance storage disks. + vmCtx.Logger.Info("Falling back into non-zonal placement since the only candidate needs host selected", + "rpMoID", candidateRPMoRefs[0].Value) + return getPlacementRecommendations(vmCtx, vcClient, candidates, configSpec) + } + + recs = append(recs, Recommendation{ + PoolMoRef: candidateRPMoRefs[0], + }) + vmCtx.Logger.V(5).Info("Implied placement since there was only one candidate", "rec", recs[0]) + + } else { + var err error + + recs, err = ClusterPlaceVMForCreate(vmCtx, vcClient, candidateRPMoRefs, configSpec, needsHost) + if err != nil { + vmCtx.Logger.Error(err, "PlaceVmsXCluster failed") + return nil + } + } + + recommendations := map[string][]Recommendation{} + for _, rec := range recs { + if rpZoneName, ok := rpMOToZone[rec.PoolMoRef]; ok { + recommendations[rpZoneName] = append(recommendations[rpZoneName], rec) + } else { + vmCtx.Logger.V(4).Info("Received unexpected ResourcePool recommendation", + "poolMoRef", rec.PoolMoRef) + } + } + + vmCtx.Logger.V(5).Info("Placement recommendations", "recommendations", recommendations) + + return recommendations +} + +// MakePlacementDecision selects one of the recommendations for placement. +func MakePlacementDecision(recommendations map[string][]Recommendation) (string, Recommendation) { + // Use an explicit rand.Intn() instead of first entry returned by map iterator. + zoneNames := make([]string, 0, len(recommendations)) + for zoneName := range recommendations { + zoneNames = append(zoneNames, zoneName) + } + zoneName := zoneNames[rand.Intn(len(zoneNames))] //nolint:gosec + + recs := recommendations[zoneName] + return zoneName, recs[rand.Intn(len(recs))] //nolint:gosec +} + +// Placement determines if the VM needs placement, and if so, determines where to place the VM +// and updates the Labels and Annotations with the placement decision. +func Placement( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + vcClient *vim25.Client, + configSpec *types.VirtualMachineConfigSpec, + childRPName string) (*Result, error) { + + existingRes, zonePlacement, instanceStoragePlacement := doesVMNeedPlacement(vmCtx) + if !zonePlacement && !instanceStoragePlacement { + return &existingRes, nil + } + + candidates, err := getPlacementCandidates(vmCtx, client, vcClient, zonePlacement, childRPName) + if err != nil { + return nil, err + } + + if len(candidates) == 0 { + return nil, fmt.Errorf("no placement candidates available") + } + + // TBD: May want to get the host for vGPU and other passthru devices too. + needsHost := instanceStoragePlacement + + var recommendations map[string][]Recommendation + if zonePlacement { + recommendations = getZonalPlacementRecommendations(vmCtx, vcClient, candidates, configSpec, needsHost) + } else /* instanceStoragePlacement */ { + recommendations = getPlacementRecommendations(vmCtx, vcClient, candidates, configSpec) + } + if len(recommendations) == 0 { + return nil, fmt.Errorf("no placement recommendations available") + } + + zoneName, rec := MakePlacementDecision(recommendations) + vmCtx.Logger.V(5).Info("Placement decision result", "zone", zoneName, "recommendation", rec) + + result := &Result{ + ZonePlacement: zonePlacement, + InstanceStoragePlacement: instanceStoragePlacement, + ZoneName: zoneName, + PoolMoRef: rec.PoolMoRef, + HostMoRef: rec.HostMoRef, + } + + return result, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go b/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go new file mode 100644 index 000000000..674b1ae86 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go @@ -0,0 +1,284 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/placement" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("MakePlacementDecision", func() { + + Context("only one placement decision is possible", func() { + It("makes expected decision", func() { + recommendations := map[string][]placement.Recommendation{ + "zone1": { + placement.Recommendation{ + PoolMoRef: types.ManagedObjectReference{Type: "a", Value: "abc"}, + HostMoRef: &types.ManagedObjectReference{Type: "b", Value: "xyz"}, + }, + }, + } + + zoneName, rec := placement.MakePlacementDecision(recommendations) + Expect(zoneName).To(Equal("zone1")) + Expect(rec).To(BeElementOf(recommendations[zoneName])) + }) + }) + + Context("multiple placement candidates exist", func() { + It("makes an decision", func() { + zones := map[string][]string{ + "zone1": {"z1-host1", "z1-host2", "z1-host3"}, + "zone2": {"z2-host1", "z2-host2"}, + } + + recommendations := map[string][]placement.Recommendation{} + for zoneName, hosts := range zones { + for _, host := range hosts { + recommendations[zoneName] = append(recommendations[zoneName], + placement.Recommendation{ + PoolMoRef: types.ManagedObjectReference{Type: "a", Value: "abc"}, + HostMoRef: &types.ManagedObjectReference{Type: "b", Value: host}, + }) + } + } + + zoneName, rec := placement.MakePlacementDecision(recommendations) + Expect(zones).To(HaveKey(zoneName)) + Expect(rec).To(BeElementOf(recommendations[zoneName])) + }) + }) +}) + +func vcSimPlacement() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + testConfig builder.VCSimTestConfig + + vm *vmopv1.VirtualMachine + vmCtx context.VirtualMachineContext + configSpec *types.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + + vm = builder.DummyVirtualMachine() + vm.Name = "placement-test" + + // Other than the name ConfigSpec contents don't matter for vcsim. + configSpec = &types.VirtualMachineConfigSpec{ + Name: vm.Name, + } + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + nsInfo = ctx.CreateWorkloadNamespace() + + vm.Namespace = nsInfo.Namespace + + vmCtx = context.VirtualMachineContext{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("Zone placement", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + Context("zone already assigned", func() { + zoneName := "in the zone" + + JustBeforeEach(func() { + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("returns success with same zone", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + Expect(result).ToNot(BeNil()) + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(Equal(zoneName)) + Expect(result.HostMoRef).To(BeNil()) + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + + // Current contract is the caller must look this up based on the pre-assigned zone but + // we might want to change that later. + Expect(result.PoolMoRef.Value).To(BeEmpty()) + }) + }) + + Context("no placement candidates", func() { + JustBeforeEach(func() { + vm.Namespace = "does-not-exist" + }) + + It("returns an error", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).To(MatchError("no placement candidates available")) + Expect(result).To(BeNil()) + }) + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + Expect(result.PoolMoRef.Value).ToNot(BeEmpty()) + Expect(result.HostMoRef).To(BeNil()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + + Context("Only one zone exists", func() { + BeforeEach(func() { + testConfig.NumFaultDomains = 1 + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + Expect(result.PoolMoRef.Value).ToNot(BeEmpty()) + Expect(result.HostMoRef).To(BeNil()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + }) + + Context("VM is in child RP via ResourcePolicy", func() { + It("returns success", func() { + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + vmCtx.VM.Spec.ResourcePolicyName = resourcePolicy.Name + + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, childRPName) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + + childRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, childRPName) + Expect(childRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(childRP.Reference().Value)) + }) + }) + }) + + Context("Instance Storage Placement", func() { + + BeforeEach(func() { + testConfig.WithInstanceStorage = true + builder.AddDummyInstanceStorageVolume(vm) + }) + + When("host already assigned", func() { + const hostMoID = "foobar-host-42" + + BeforeEach(func() { + vm.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey] = hostMoID + }) + + It("returns success with same host", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).To(Equal(hostMoID)) + }) + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + }) + + When("FaultDomains FSS is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + testConfig.NumFaultDomains = 1 // Only support for non-HA "HA" + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).ToNot(BeEmpty()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + + Context("VM is in child RP via ResourcePolicy", func() { + It("returns success", func() { + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + vmCtx.VM.Spec.ResourcePolicyName = resourcePolicy.Name + + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, childRPName) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + + childRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, childRPName) + Expect(childRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(childRP.Reference().Value)) + }) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/resources/vm.go b/pkg/vmprovider/providers/vsphere2/resources/vm.go new file mode 100644 index 000000000..ddc90e713 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/resources/vm.go @@ -0,0 +1,230 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" +) + +type VirtualMachine struct { + Name string + vcVirtualMachine *object.VirtualMachine + logger logr.Logger +} + +var log = logf.Log.WithName("vmresource") + +func NewVMFromObject(objVM *object.VirtualMachine) *VirtualMachine { + return &VirtualMachine{ + Name: objVM.Name(), + vcVirtualMachine: objVM, + logger: log.WithValues("name", objVM.Name()), + } +} + +func (vm *VirtualMachine) VcVM() *object.VirtualMachine { + return vm.vcVirtualMachine +} + +func (vm *VirtualMachine) Create(ctx context.Context, folder *object.Folder, pool *object.ResourcePool, vmSpec *types.VirtualMachineConfigSpec) error { + vm.logger.V(5).Info("Create VM") + + if vm.vcVirtualMachine != nil { + return fmt.Errorf("failed to create VM %q because the VM object is already set", vm.Name) + } + + createTask, err := folder.CreateVM(ctx, *vmSpec, pool, nil) + if err != nil { + return err + } + + result, err := createTask.WaitForResult(ctx, nil) + if err != nil { + return errors.Wrapf(err, "create VM %q task failed", vm.Name) + } + + vm.vcVirtualMachine = object.NewVirtualMachine(folder.Client(), result.Result.(types.ManagedObjectReference)) + return nil +} + +func (vm *VirtualMachine) Clone(ctx context.Context, folder *object.Folder, cloneSpec *types.VirtualMachineCloneSpec) (*types.ManagedObjectReference, error) { + vm.logger.V(5).Info("Clone VM") + + cloneTask, err := vm.vcVirtualMachine.Clone(ctx, folder, cloneSpec.Config.Name, *cloneSpec) + if err != nil { + return nil, err + } + + result, err := cloneTask.WaitForResult(ctx, nil) + if err != nil { + return nil, errors.Wrapf(err, "clone VM task failed") + } + + ref := result.Result.(types.ManagedObjectReference) + return &ref, nil +} + +func (vm *VirtualMachine) Reconfigure(ctx context.Context, configSpec *types.VirtualMachineConfigSpec) error { + vm.logger.V(5).Info("Reconfiguring VM", "configSpec", configSpec) + + reconfigureTask, err := vm.vcVirtualMachine.Reconfigure(ctx, *configSpec) + if err != nil { + return err + } + + _, err = reconfigureTask.WaitForResult(ctx, nil) + if err != nil { + return errors.Wrapf(err, "reconfigure VM task failed") + } + + return nil +} + +func (vm *VirtualMachine) GetProperties(ctx context.Context, properties []string) (*mo.VirtualMachine, error) { + var o mo.VirtualMachine + err := vm.vcVirtualMachine.Properties(ctx, vm.vcVirtualMachine.Reference(), properties, &o) + if err != nil { + vm.logger.Error(err, "Error getting VM properties") + return nil, err + } + + return &o, nil +} + +func (vm *VirtualMachine) ReferenceValue() string { + vm.logger.V(5).Info("Get ReferenceValue") + return vm.vcVirtualMachine.Reference().Value +} + +func (vm *VirtualMachine) MoRef() types.ManagedObjectReference { + vm.logger.V(5).Info("Get MoRef") + return vm.vcVirtualMachine.Reference() +} + +func (vm *VirtualMachine) UniqueID(ctx context.Context) (string, error) { + // Notes from Alkesh Shah regarding MoIDs in VC as of 7.0 + // + // MoRef IDs are unique within the scope of a single VC. Since Clusters are entities in VCs, the MoRef IDs will be unique across clusters + // + // Identity in VC is derived from a sequence. This ID is used in generating the MoId (or MoRef ID) for the entity in VC. Sequence is monotonically + // increasing and so during regular operation there are no dupes + // + // Backup-Restore: We now make sure that our sequence does not go back in time when restoring from a backup + // ( ) So this + // ensures that after restore we get new MoIds which are never used before… (we advance the sequence counter based on time) + // + // Discovery of VMs: We only use moids from the VM store during restore from a backup. In the unlikely event + // that there are two VMs which are presenting the same MoId, we will regenerate a new MoId based on the current + // sequence. Keep in mind, Ideally the unlikely scenario should not occur as we attempt to tamper proof the MoId + // stored in the VM store ( ) + // so two VMs having the same MoId should not happen because they cannot have the same VMX path and we use VMX path + // for ensuring this tamper proof behavior. + // + // Removing from VC and Re-adding the VM to same VC: VM will be given a new MoId (even if the VM is added using + // RegisterVM operation from VC) + // Basically, lifetime of the identity is tied to VC’s knowledge of it’s existence in it’s inventory + return vm.ReferenceValue(), nil +} + +func (vm *VirtualMachine) SetPowerState( + ctx context.Context, + currentPowerState, + desiredPowerState vmopv1.VirtualMachinePowerState, + desiredPowerOpMode vmopv1.VirtualMachinePowerOpMode) error { + + _, err := vmutil.SetAndWaitOnPowerState( + ctx, + vm.VcVM().Client(), + mo.VirtualMachine{ + ManagedEntity: mo.ManagedEntity{ + ExtensibleManagedObject: mo.ExtensibleManagedObject{ + Self: vm.VcVM().Reference(), + }, + }, + Summary: types.VirtualMachineSummary{ + Runtime: types.VirtualMachineRuntimeInfo{ + PowerState: vmutil.ParsePowerState(string(currentPowerState)), + }, + }, + }, + false, + vmutil.ParsePowerState(string(desiredPowerState)), + vmutil.ParsePowerOpMode(string(desiredPowerOpMode))) + + return err +} + +// GetVirtualDevices returns the VMs VirtualDeviceList. +func (vm *VirtualMachine) GetVirtualDevices(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(5).Info("GetVirtualDevices") + deviceList, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return deviceList, err +} + +// GetVirtualDisks returns the list of VMs vmdks. +func (vm *VirtualMachine) GetVirtualDisks(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(5).Info("GetVirtualDisks") + deviceList, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return deviceList.SelectByType((*types.VirtualDisk)(nil)), nil +} + +func (vm *VirtualMachine) GetNetworkDevices(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(4).Info("GetNetworkDevices") + devices, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return devices.SelectByType((*types.VirtualEthernetCard)(nil)), nil +} + +func (vm *VirtualMachine) Customize(ctx context.Context, spec types.CustomizationSpec) error { + vm.logger.V(5).Info("Customize", "spec", spec) + + customizeTask, err := vm.vcVirtualMachine.Customize(ctx, spec) + if err != nil { + vm.logger.Error(err, "Failed to customize VM") + return err + } + + taskInfo, err := customizeTask.WaitForResult(ctx, nil) + if err != nil { + vm.logger.Error(err, "Failed to wait for the result of Customize VM") + return err + } + + if taskErr := taskInfo.Error; taskErr != nil { + // Fetch fault messages for task.Error + fault := taskErr.Fault.GetMethodFault() + if fault != nil { + err = errors.Wrapf(err, "Fault messages: %v", fault.FaultMessage) + } + + return errors.Wrap(err, "Customization task failed") + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session.go b/pkg/vmprovider/providers/vsphere2/session/session.go new file mode 100644 index 000000000..ea0eb9881 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session.go @@ -0,0 +1,41 @@ +// Copyright (c) 2018-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" +) + +type Session struct { + Client *client.Client + K8sClient ctrlruntime.Client + Finder *find.Finder + + // Fields only used during Update + Cluster *object.ClusterComputeResource +} + +func (s *Session) invokeFsrVirtualMachine(vmCtx context.VirtualMachineContextA2, resVM *res.VirtualMachine) error { + vmCtx.Logger.Info("Invoking FSR on VM") + + task, err := internal.VirtualMachineFSR(vmCtx, resVM.MoRef(), s.Client.VimClient()) + if err != nil { + vmCtx.Logger.Error(err, "InvokeFSR call failed") + return err + } + + if err = task.Wait(vmCtx); err != nil { + vmCtx.Logger.Error(err, "InvokeFSR task failed") + return err + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go b/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go new file mode 100644 index 000000000..011bbe5b1 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestSession(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Session Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util.go b/pkg/vmprovider/providers/vsphere2/session/session_util.go new file mode 100644 index 000000000..fb25dbe91 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_util.go @@ -0,0 +1,34 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + vimTypes "github.com/vmware/govmomi/vim25/types" +) + +func ExtraConfigToMap(input []vimTypes.BaseOptionValue) (output map[string]string) { + output = make(map[string]string) + for _, opt := range input { + if optValue := opt.GetOptionValue(); optValue != nil { + // Only set string type values + if val, ok := optValue.Value.(string); ok { + output[optValue.Key] = val + } + } + } + return +} + +// MergeExtraConfig adds the key/value to the ExtraConfig if the key is not present, to let to the value be +// changed by the VM. The existing usage of ExtraConfig is hard to fit in the reconciliation model. +func MergeExtraConfig(extraConfig []vimTypes.BaseOptionValue, newMap map[string]string) []vimTypes.BaseOptionValue { + merged := make([]vimTypes.BaseOptionValue, 0) + ecMap := ExtraConfigToMap(extraConfig) + for k, v := range newMap { + if _, exists := ecMap[k]; !exists { + merged = append(merged, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + return merged +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util_test.go b/pkg/vmprovider/providers/vsphere2/session/session_util_test.go new file mode 100644 index 000000000..76ca2a0fb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_util_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2019-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" +) + +var _ = Describe("Test Session Utils", func() { + + Context("ExtraConfigToMap", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + extraConfigMap map[string]string + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{} + }) + JustBeforeEach(func() { + extraConfigMap = session.ExtraConfigToMap(extraConfig) + }) + + Context("Empty extraConfig", func() { + It("Return empty map", func() { + Expect(extraConfigMap).To(HaveLen(0)) + }) + }) + + Context("With extraConfig", func() { + BeforeEach(func() { + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key1", Value: "value1"}) + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key2", Value: "value2"}) + }) + It("Return valid map", func() { + Expect(extraConfigMap).To(HaveLen(2)) + Expect(extraConfigMap["key1"]).To(Equal("value1")) + Expect(extraConfigMap["key2"]).To(Equal("value2")) + }) + }) + }) + + Context("MergeExtraConfig", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + newMap map[string]string + merged []vimTypes.BaseOptionValue + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{Key: "existingkey1", Value: "existingvalue1"}, + &vimTypes.OptionValue{Key: "existingkey2", Value: "existingvalue2"}, + } + newMap = map[string]string{} + }) + JustBeforeEach(func() { + merged = session.MergeExtraConfig(extraConfig, newMap) + }) + + Context("Empty newMap", func() { + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with existing key", func() { + BeforeEach(func() { + newMap["existingkey1"] = "existingkey1" + }) + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with new keys", func() { + BeforeEach(func() { + newMap["newkey1"] = "newvalue1" + newMap["newkey2"] = "newvalue2" + }) + It("Return merged map", func() { + Expect(merged).To(HaveLen(2)) + mergedMap := session.ExtraConfigToMap(merged) + Expect(mergedMap["newkey1"]).To(Equal("newvalue1")) + Expect(mergedMap["newkey2"]).To(Equal("newvalue2")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm.go b/pkg/vmprovider/providers/vsphere2/session/session_vm.go new file mode 100644 index 000000000..a7aaff105 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "fmt" + + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +func updateVirtualDiskDeviceChanges( + vmCtx context.VirtualMachineContextA2, + virtualDisks object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil, nil + } + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + found := false + for _, vmDevice := range virtualDisks { + vmDisk, ok := vmDevice.(*vimTypes.VirtualDisk) + if !ok { + continue + } + + // Assume the first disk as the boot disk. We can make this smarter by + // looking at the disk path or whatever else later. + // TODO: De-dupe this with resizeBootDiskDeviceChange() in the clone path. + + newCapacityInBytes := capacity.Value() + if newCapacityInBytes < vmDisk.CapacityInBytes { + err := fmt.Errorf("cannot shrink boot disk from %d bytes to %d bytes", + vmDisk.CapacityInBytes, newCapacityInBytes) + return nil, err + } + + if vmDisk.CapacityInBytes < newCapacityInBytes { + vmDisk.CapacityInBytes = newCapacityInBytes + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationEdit, + Device: vmDisk, + }) + } + + found = true + break + } + + if !found { + return nil, fmt.Errorf("could not find the boot disk to change capacity") + } + + return deviceChanges, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go new file mode 100644 index 000000000..1154c20fa --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -0,0 +1,957 @@ +// Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "fmt" + "reflect" + "time" + + "k8s.io/utils/pointer" + + "github.com/go-logr/logr" + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + apiEquality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/util" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + network2 "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +const ( + FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" +) + +// VMUpdateArgs contains the arguments needed to update a VM on VC. +type VMUpdateArgs struct { + VMClass *vmopv1.VirtualMachineClass + ResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + MinCPUFreq uint64 + ExtraConfig map[string]string + + BootstrapData vmlifecycle.BootstrapData + + ConfigSpec *vimTypes.VirtualMachineConfigSpec + + NetworkResults network2.NetworkInterfaceResults + + // hack. Remove after VMSVC-1261. + // indicating if this VM image used is VM service v1alpha1 compatible. + VirtualMachineImageV1Alpha1Compatible bool +} + +func ethCardMatch(newBaseEthCard, curBaseEthCard vimTypes.BaseVirtualEthernetCard) bool { + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if reflect.TypeOf(curBaseEthCard) != reflect.TypeOf(newBaseEthCard) { + return false + } + } + + curEthCard := curBaseEthCard.GetVirtualEthernetCard() + newEthCard := newBaseEthCard.GetVirtualEthernetCard() + if newEthCard.AddressType == string(vimTypes.VirtualEthernetCardMacTypeManual) { + // If the new card has an assigned MAC address, then it should match with + // the current card. Note only NCP sets the MAC address. + if newEthCard.MacAddress != curEthCard.MacAddress { + return false + } + } + + if newEthCard.ExternalId != "" { + // If the new card has a specific ExternalId, then it should match with the + // current card. Note only NCP sets the ExternalId. + if newEthCard.ExternalId != curEthCard.ExternalId { + return false + } + } + + return true +} + +func UpdateEthCardDeviceChanges( + expectedEthCards object.VirtualDeviceList, + currentEthCards object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + for _, expectedDev := range expectedEthCards { + expectedNic := expectedDev.(vimTypes.BaseVirtualEthernetCard) + expectedBacking := expectedNic.GetVirtualEthernetCard().Backing + expectedBackingType := reflect.TypeOf(expectedBacking) + + var matchingIdx = -1 + + // Try to match the expected NIC with an existing NIC but this isn't that great. We mostly + // depend on the backing but we can improve that later on. When not generated, we could use + // the MAC address. When we support something other than just vmxnet3 we should compare + // those types too. And we should make this truly reconcile as well by comparing the full + // state (support EDIT instead of only ADD/REMOVE operations). + // + // Another tack we could take is force the VM's device order to match the Spec order, but + // that could lead to spurious removals. Or reorder the NetIfList to not be that of the + // Spec, but in VM device order. + for idx, curDev := range currentEthCards { + nic := curDev.(vimTypes.BaseVirtualEthernetCard) + + // This assumes we don't have multiple NICs in the same backing network. This is kind of, sort + // of enforced by the webhook, but we lack a guaranteed way to match up the NICs. + + if !ethCardMatch(expectedNic, nic) { + continue + } + + db := nic.GetVirtualEthernetCard().Backing + if db == nil || reflect.TypeOf(db) != expectedBackingType { + continue + } + + var backingMatch bool + + // Cribbed from VirtualDeviceList.SelectByBackingInfo(). + switch a := db.(type) { + case *vimTypes.VirtualEthernetCardNetworkBackingInfo: + // This backing is only used in testing. + b := expectedBacking.(*vimTypes.VirtualEthernetCardNetworkBackingInfo) + backingMatch = a.DeviceName == b.DeviceName + case *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo: + b := expectedBacking.(*vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo) + backingMatch = a.Port.SwitchUuid == b.Port.SwitchUuid && a.Port.PortgroupKey == b.Port.PortgroupKey + case *vimTypes.VirtualEthernetCardOpaqueNetworkBackingInfo: + b := expectedBacking.(*vimTypes.VirtualEthernetCardOpaqueNetworkBackingInfo) + backingMatch = a.OpaqueNetworkId == b.OpaqueNetworkId + } + + if backingMatch { + matchingIdx = idx + break + } + } + + if matchingIdx == -1 { + // No matching backing found so add new card. + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: expectedDev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + }) + } else { + // Matching backing found so keep this card (don't remove it below after this loop). + currentEthCards = append(currentEthCards[:matchingIdx], currentEthCards[matchingIdx+1:]...) + } + } + + // Remove any unmatched existing interfaces. + removeDeviceChanges := make([]vimTypes.BaseVirtualDeviceConfigSpec, 0, len(currentEthCards)) + for _, dev := range currentEthCards { + removeDeviceChanges = append(removeDeviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: dev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationRemove, + }) + } + + // Process any removes first. + return append(removeDeviceChanges, deviceChanges...), nil +} + +// UpdatePCIDeviceChanges returns devices changes for PCI devices attached to a VM. There are 2 types of PCI devices +// processed here and in case of cloning a VM, devices listed in VMClass are considered as source of truth. +func UpdatePCIDeviceChanges( + expectedPciDevices object.VirtualDeviceList, + currentPciDevices object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + for _, expectedDev := range expectedPciDevices { + expectedPci := expectedDev.(*vimTypes.VirtualPCIPassthrough) + expectedBacking := expectedPci.Backing + expectedBackingType := reflect.TypeOf(expectedBacking) + + var matchingIdx = -1 + for idx, curDev := range currentPciDevices { + curBacking := curDev.GetVirtualDevice().Backing + if curBacking == nil || reflect.TypeOf(curBacking) != expectedBackingType { + continue + } + + var backingMatch bool + switch a := curBacking.(type) { + case *vimTypes.VirtualPCIPassthroughVmiopBackingInfo: + b := expectedBacking.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo) + backingMatch = a.Vgpu == b.Vgpu + + case *vimTypes.VirtualPCIPassthroughDynamicBackingInfo: + currAllowedDevs := a.AllowedDevice + b := expectedBacking.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo) + if a.CustomLabel == b.CustomLabel { + // b.AllowedDevice has only one element because CreatePCIDevices() adds only one device based + // on the devices listed in vmclass.spec.hardware.devices.dynamicDirectPathIODevices. + expectedAllowedDev := b.AllowedDevice[0] + for i := 0; i < len(currAllowedDevs) && !backingMatch; i++ { + backingMatch = expectedAllowedDev.DeviceId == currAllowedDevs[i].DeviceId && + expectedAllowedDev.VendorId == currAllowedDevs[i].VendorId + } + } + } + + if backingMatch { + matchingIdx = idx + break + } + } + + if matchingIdx == -1 { + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: expectedPci, + }) + } else { + // There could be multiple vGPUs with same BackingInfo. Remove current device if matching found. + currentPciDevices = append(currentPciDevices[:matchingIdx], currentPciDevices[matchingIdx+1:]...) + } + } + // Remove any unmatched existing devices. + removeDeviceChanges := make([]vimTypes.BaseVirtualDeviceConfigSpec, 0, len(currentPciDevices)) + for _, dev := range currentPciDevices { + removeDeviceChanges = append(removeDeviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: dev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationRemove, + }) + } + + // Process any removes first. + return append(removeDeviceChanges, deviceChanges...), nil +} + +func UpdateConfigSpecCPUAllocation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + minCPUFreq uint64) { + + cpuAllocation := config.CpuAllocation + var cpuReservation *int64 + var cpuLimit *int64 + + if !vmClassSpec.Policies.Resources.Requests.Cpu.IsZero() { + rsv := virtualmachine.CPUQuantityToMhz(vmClassSpec.Policies.Resources.Requests.Cpu, minCPUFreq) + if cpuAllocation == nil || cpuAllocation.Reservation == nil || *cpuAllocation.Reservation != rsv { + cpuReservation = &rsv + } + } + + if !vmClassSpec.Policies.Resources.Limits.Cpu.IsZero() { + lim := virtualmachine.CPUQuantityToMhz(vmClassSpec.Policies.Resources.Limits.Cpu, minCPUFreq) + if cpuAllocation == nil || cpuAllocation.Limit == nil || *cpuAllocation.Limit != lim { + cpuLimit = &lim + } + } + + if cpuReservation != nil || cpuLimit != nil { + configSpec.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: cpuReservation, + Limit: cpuLimit, + } + } +} + +func UpdateConfigSpecMemoryAllocation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec) { + + memAllocation := config.MemoryAllocation + var memoryReservation *int64 + var memoryLimit *int64 + + if !vmClassSpec.Policies.Resources.Requests.Memory.IsZero() { + rsv := virtualmachine.MemoryQuantityToMb(vmClassSpec.Policies.Resources.Requests.Memory) + if memAllocation == nil || memAllocation.Reservation == nil || *memAllocation.Reservation != rsv { + memoryReservation = &rsv + } + } + + if !vmClassSpec.Policies.Resources.Limits.Memory.IsZero() { + lim := virtualmachine.MemoryQuantityToMb(vmClassSpec.Policies.Resources.Limits.Memory) + if memAllocation == nil || memAllocation.Limit == nil || *memAllocation.Limit != lim { + memoryLimit = &lim + } + } + + if memoryReservation != nil || memoryLimit != nil { + configSpec.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: memoryReservation, + Limit: memoryLimit, + } + } +} + +func UpdateConfigSpecExtraConfig( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + vm *vmopv1.VirtualMachine, + globalExtraConfig map[string]string, + imageV1Alpha1Compatible bool) { + + extraConfig := make(map[string]string) + for k, v := range globalExtraConfig { + extraConfig[k] = v + } + + virtualDevices := vmClassSpec.Hardware.Devices + pciPassthruFromConfigSpec := util.SelectVirtualPCIPassthrough(util.DevicesFromConfigSpec(classConfigSpec)) + if len(virtualDevices.VGPUDevices) > 0 || len(virtualDevices.DynamicDirectPathIODevices) > 0 || len(pciPassthruFromConfigSpec) > 0 { + // Add "maintenance.vm.evacuation.poweroff" extraConfig key when GPU devices are present in the VMClass Spec. + extraConfig[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + setMMIOExtraConfig(vm, extraConfig) + } + + // If VM has InstanceStorage configured, add "maintenance.vm.evacuation.poweroff" to extraConfig + if instancestorage.IsPresent(vm) { + extraConfig[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + // Merge non intersecting keys from the desired config spec extra config with the class config spec extra config + // (ie) class config spec extra config keys takes precedence over the desired config spec extra config keys + ecFromClassConfigSpec := ExtraConfigToMap(classConfigSpec.ExtraConfig) + mergedExtraConfig := classConfigSpec.ExtraConfig + for k, v := range extraConfig { + if _, exists := ecFromClassConfigSpec[k]; !exists { + mergedExtraConfig = append(mergedExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + extraConfig = ExtraConfigToMap(mergedExtraConfig) + } + + configSpec.ExtraConfig = MergeExtraConfig(config.ExtraConfig, extraConfig) + + // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot + // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled + // when the vmMetadata is nil or when the transport requested is not CloudInit. + // VMSVC-1261: we may always set this extra config key to remove image from VM customization. + // If a VM is deployed from an incompatible image, + // it will do nothing and won't cause any issues, but can introduce confusion. + // BMV: Is this needed anymore? IMO we shouldn't have bootstrap stuff here. The EC mangling is already hard to follow. + emptyBSSpec := vmopv1.VirtualMachineBootstrapSpec{} + if vm.Spec.Bootstrap == emptyBSSpec || vm.Spec.Bootstrap.CloudInit == nil { + ecMap := ExtraConfigToMap(config.ExtraConfig) + if ecMap[constants.VMOperatorV1Alpha1ExtraConfigKey] == constants.VMOperatorV1Alpha1ConfigReady && + imageV1Alpha1Compatible { + // Set VMOperatorV1Alpha1ExtraConfigKey for v1alpha1 VirtualMachineImage compatibility. + configSpec.ExtraConfig = append(configSpec.ExtraConfig, + &vimTypes.OptionValue{Key: constants.VMOperatorV1Alpha1ExtraConfigKey, Value: constants.VMOperatorV1Alpha1ConfigEnabled}) + } + } +} + +func setMMIOExtraConfig(vm *vmopv1.VirtualMachine, extraConfig map[string]string) { + mmioSize := vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] + if mmioSize == "" { + mmioSize = constants.PCIPassthruMMIOSizeDefault + } + if mmioSize != "0" { + extraConfig[constants.PCIPassthruMMIOExtraConfigKey] = constants.ExtraConfigTrue + extraConfig[constants.PCIPassthruMMIOSizeExtraConfigKey] = mmioSize + } +} + +func UpdateConfigSpecChangeBlockTracking( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec, + vmSpec vmopv1.VirtualMachineSpec) { + + // When VM_Class_as_Config_DaynDate is enabled, class config spec cbt if + // set overrides the VM spec advanced options cbt. + // BMV: I don't think this is correct: the class shouldn't dictate this for backup purposes. There is a + // webhook out there that changes this in the VM spec. + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && classConfigSpec != nil { + if classConfigSpec.ChangeTrackingEnabled != nil { + if !apiEquality.Semantic.DeepEqual(config.ChangeTrackingEnabled, classConfigSpec.ChangeTrackingEnabled) { + configSpec.ChangeTrackingEnabled = classConfigSpec.ChangeTrackingEnabled + } + return + } + } + + if vmSpec.Advanced.ChangeBlockTracking { + if config.ChangeTrackingEnabled == nil || !*config.ChangeTrackingEnabled { + configSpec.ChangeTrackingEnabled = pointer.Bool(true) + } + } else { + if config.ChangeTrackingEnabled != nil && *config.ChangeTrackingEnabled { + configSpec.ChangeTrackingEnabled = pointer.Bool(false) + } + } +} + +func UpdateHardwareConfigSpec( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec) { + + if nCPUs := int32(vmClassSpec.Hardware.Cpus); config.Hardware.NumCPU != nCPUs { + configSpec.NumCPUs = nCPUs + } + if memMB := virtualmachine.MemoryQuantityToMb(vmClassSpec.Hardware.Memory); int64(config.Hardware.MemoryMB) != memMB { + configSpec.MemoryMB = memMB + } +} + +func UpdateConfigSpecAnnotation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec) { + if config.Annotation != constants.VCVMAnnotation { + configSpec.Annotation = constants.VCVMAnnotation + } +} + +func UpdateConfigSpecManagedBy( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec) { + if config.ManagedBy == nil { + configSpec.ManagedBy = &vimTypes.ManagedByInfo{ + ExtensionKey: constants.ManagedByExtensionKey, + Type: constants.ManagedByExtensionType, + } + } +} + +func UpdateConfigSpecFirmware( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vm *vmopv1.VirtualMachine) { + + if val, ok := vm.Annotations[constants.FirmwareOverrideAnnotation]; ok { + if (val == "efi" || val == "bios") && config.Firmware != val { + configSpec.Firmware = val + } + } +} + +// UpdateConfigSpecDeviceGroups sets the desired config spec device groups to reconcile by differencing the +// current VM config and the class config spec device groups. +func UpdateConfigSpecDeviceGroups( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec) { + + if classConfigSpec.DeviceGroups != nil { + if config.DeviceGroups == nil || !reflect.DeepEqual(classConfigSpec.DeviceGroups.DeviceGroup, config.DeviceGroups.DeviceGroup) { + configSpec.DeviceGroups = classConfigSpec.DeviceGroups + } + } +} + +// updateConfigSpec overlays the VM Class spec with the provided ConfigSpec to form a desired +// ConfigSpec that will be used to reconfigure the VM. +func updateConfigSpec( + vmCtx context.VirtualMachineContextA2, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) *vimTypes.VirtualMachineConfigSpec { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + vmClassSpec := updateArgs.VMClass.Spec + + // Before VM Class as Config, VMs were deployed from the OVA, and are then + // reconfigured to match the desired CPU and memory reservation. Maintain that + // behavior. With the FSS enabled, VMs will be _created_ with desired HW spec, and we + // will not modify the hardware of the VM post creation. So, don't populate the + // Hardware config and CPU/Memory reservation. + if !lib.IsVMClassAsConfigFSSDaynDateEnabled() { + UpdateHardwareConfigSpec(config, configSpec, &vmClassSpec) + UpdateConfigSpecCPUAllocation(config, configSpec, &vmClassSpec, updateArgs.MinCPUFreq) + UpdateConfigSpecMemoryAllocation(config, configSpec, &vmClassSpec) + } + + UpdateConfigSpecAnnotation(config, configSpec) + UpdateConfigSpecManagedBy(config, configSpec) + UpdateConfigSpecExtraConfig(config, configSpec, updateArgs.ConfigSpec, &vmClassSpec, + vmCtx.VM, updateArgs.ExtraConfig, updateArgs.VirtualMachineImageV1Alpha1Compatible) + UpdateConfigSpecChangeBlockTracking(config, configSpec, updateArgs.ConfigSpec, vmCtx.VM.Spec) + UpdateConfigSpecFirmware(config, configSpec, vmCtx.VM) + UpdateConfigSpecDeviceGroups(config, configSpec, updateArgs.ConfigSpec) + + return configSpec +} + +func (s *Session) prePowerOnVMConfigSpec( + vmCtx context.VirtualMachineContextA2, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) (*vimTypes.VirtualMachineConfigSpec, error) { + + configSpec := updateConfigSpec(vmCtx, config, updateArgs) + + virtualDevices := object.VirtualDeviceList(config.Hardware.Device) + currentDisks := virtualDevices.SelectByType((*vimTypes.VirtualDisk)(nil)) + currentEthCards := virtualDevices.SelectByType((*vimTypes.VirtualEthernetCard)(nil)) + currentPciDevices := virtualDevices.SelectByType((*vimTypes.VirtualPCIPassthrough)(nil)) + + diskDeviceChanges, err := updateVirtualDiskDeviceChanges(vmCtx, currentDisks) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, diskDeviceChanges...) + + var expectedEthCards object.VirtualDeviceList + for idx := range updateArgs.NetworkResults.Results { + expectedEthCards = append(expectedEthCards, updateArgs.NetworkResults.Results[idx].Device) + } + + ethCardDeviceChanges, err := UpdateEthCardDeviceChanges(expectedEthCards, currentEthCards) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, ethCardDeviceChanges...) + + var expectedPCIDevices []vimTypes.BaseVirtualDevice + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if configSpecDevs := util.DevicesFromConfigSpec(updateArgs.ConfigSpec); len(configSpecDevs) > 0 { + pciPassthruFromConfigSpec := util.SelectVirtualPCIPassthrough(configSpecDevs) + expectedPCIDevices = virtualmachine.CreatePCIDevicesFromConfigSpec(pciPassthruFromConfigSpec) + } + } else { + expectedPCIDevices = virtualmachine.CreatePCIDevicesFromVMClass(updateArgs.VMClass.Spec.Hardware.Devices) + } + + pciDeviceChanges, err := UpdatePCIDeviceChanges(expectedPCIDevices, currentPciDevices) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, pciDeviceChanges...) + + return configSpec, nil +} + +func (s *Session) prePowerOnVMReconfigure( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + configSpec, err := s.prePowerOnVMConfigSpec(vmCtx, config, updateArgs) + if err != nil { + return err + } + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("Pre PowerOn Reconfigure", "configSpec", configSpec) + if err := resVM.Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "pre power on reconfigure failed") + return err + } + } + + return nil +} + +func (s *Session) ensureNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + configSpec *vimTypes.VirtualMachineConfigSpec) (network2.NetworkInterfaceResults, error) { + + // This negative device key is the traditional range used for network interfaces. + deviceKey := int32(-100) + + var networkDevices []vimTypes.BaseVirtualDevice + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && configSpec != nil { + networkDevices = util.SelectDevicesByTypes( + util.DevicesFromConfigSpec(configSpec), + &vimTypes.VirtualE1000{}, + &vimTypes.VirtualE1000e{}, + &vimTypes.VirtualPCNet32{}, + &vimTypes.VirtualVmxnet2{}, + &vimTypes.VirtualVmxnet3{}, + &vimTypes.VirtualVmxnet3Vrdma{}, + &vimTypes.VirtualSriovEthernetCard{}, + ) + } + networkSpec := &vmCtx.VM.Spec.Network + if networkSpec.Disabled { + // No connected networking for this VM. + return network2.NetworkInterfaceResults{}, nil + } + + interfaces := networkSpec.Interfaces + if len(interfaces) == 0 { + // VM gets one automatic NIC. Create the default interface from fields in the network spec. + defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: networkSpec.DeviceName, + Addresses: networkSpec.Addresses, + DHCP4: networkSpec.DHCP4, + DHCP6: networkSpec.DHCP6, + Gateway4: networkSpec.Gateway4, + Gateway6: networkSpec.Gateway6, + MTU: networkSpec.MTU, + Nameservers: networkSpec.Nameservers, + Routes: networkSpec.Routes, + SearchDomains: networkSpec.SearchDomains, + } + + if defaultInterface.Name == "" { + defaultInterface.Name = "eth0" + } + if networkSpec.Network != nil { + defaultInterface.Network = *networkSpec.Network + } + + interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} + } + + clusterMoRef := s.Cluster.Reference() + results, err := network2.CreateAndWaitForNetworkInterfaces( + vmCtx, + s.K8sClient, + s.Client.VimClient(), + s.Finder, + &clusterMoRef, + interfaces) + if err != nil { + return network2.NetworkInterfaceResults{}, err + } + + // XXX: The following logic assumes that the order of network interfaces specified in the + // VM spec matches one to one with the device changes in the ConfigSpec in VM class. + // This is a safe assumption for now since VM service only supports one network interface. + // TODO: Needs update when VM Service supports VMs with more then one network interface. + for idx := range results.Results { + result := &results.Results[idx] + + dev, err := network2.CreateDefaultEthCard(vmCtx, result) + if err != nil { + return network2.NetworkInterfaceResults{}, err + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + // If VM Class-as-a-Config is supported, we use the network device from the Class. + // If the VM class doesn't specify enough number of network devices, we fall back to default behavior. + if idx < len(networkDevices) { + ethCardFromNetProvider := dev.(vimTypes.BaseVirtualEthernetCard) + + if mac := ethCardFromNetProvider.GetVirtualEthernetCard().MacAddress; mac != "" { + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = mac + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + } + + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = + ethCardFromNetProvider.GetVirtualEthernetCard().ExternalId + // If the device from VM class has a DVX backing, this should still work if the backing as well + // as the DVX backing are set. VPXD checks for DVX backing before checking for normal device backings. + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().Backing = + ethCardFromNetProvider.GetVirtualEthernetCard().Backing + + dev = networkDevices[idx] + } + } + + // govmomi assigns a random device key. Fix that up here. + dev.GetVirtualDevice().Key = deviceKey + deviceKey-- + + result.Device = dev + } + + return results, nil +} + +func (s *Session) ensureCNSVolumes(vmCtx context.VirtualMachineContextA2) error { + // If VM spec has a PVC, check if the volume is attached before powering on + for _, volume := range vmCtx.VM.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + // Only handle PVC volumes here. In v1a1 we had non-PVC ("vsphereVolumes") but those are gone. + continue + } + + // BMV: We should not use the Status as the SoT here. What a mess. + found := false + for _, volumeStatus := range vmCtx.VM.Status.Volumes { + if volumeStatus.Name == volume.Name { + found = true + if !volumeStatus.Attached { + return fmt.Errorf("persistent volume: %s not attached to VM", volume.Name) + } + break + } + } + + if !found { + return fmt.Errorf("status update pending for persistent volume: %s on VM", volume.Name) + } + } + + return nil +} + +func (s *Session) customize( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + cfg *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + { + // Hack: the old code only populated the first nic address - getFirstNicMacAddr() - so just keep + // doing the same here. We need a generalized UpdateEthCardDeviceChanges() to match up the Spec + // with the actual devices. Old code also made this best effort so do that here too. + // I've got a larger change that removes the old session stuff, and improves on all this behavior + // but I didn't have the BW to sort out all the changes. + if len(updateArgs.NetworkResults.Results) > 1 { + mac := updateArgs.NetworkResults.Results[0].MacAddress + if mac == "" { + ethCards, _ := resVM.GetNetworkDevices(vmCtx) + if len(ethCards) > 0 { + curNic := ethCards[0].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard() + updateArgs.NetworkResults.Results[0].MacAddress = curNic.GetVirtualEthernetCard().MacAddress + } + } + } + } + + err := vmlifecycle.DoBootstrap(vmCtx, resVM.VcVM(), cfg, s.K8sClient, updateArgs.NetworkResults, updateArgs.BootstrapData) + if err != nil { + return err + } + + return nil +} + +func (s *Session) prepareVMForPowerOn( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + cfg *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + netIfList, err := s.ensureNetworkInterfaces(vmCtx, updateArgs.ConfigSpec) + if err != nil { + return err + } + + updateArgs.NetworkResults = netIfList + + err = s.prePowerOnVMReconfigure(vmCtx, resVM, cfg, updateArgs) + if err != nil { + return err + } + + err = s.customize(vmCtx, resVM, cfg, updateArgs) + if err != nil { + return err + } + + err = s.ensureCNSVolumes(vmCtx) + if err != nil { + return err + } + + return nil +} + +func (s *Session) poweredOnVMReconfigure( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo) error { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + UpdateConfigSpecChangeBlockTracking(config, configSpec, nil, vmCtx.VM.Spec) + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("PoweredOn Reconfigure", "configSpec", configSpec) + if err := resVM.Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "powered on reconfigure failed") + return err + } + + // Special case for CBT: in order for CBT change take effect for a powered on VM, + // a checkpoint save/restore is needed. tracks the implementation of + // this FSR internally to vSphere. + if configSpec.ChangeTrackingEnabled != nil { + if err := s.invokeFsrVirtualMachine(vmCtx, resVM); err != nil { + vmCtx.Logger.Error(err, "Failed to invoke FSR for CBT update") + return err + } + } + } + + return nil +} + +func (s *Session) attachClusterModule( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + // The clusterModule is required be able to enforce the vm-vm anti-affinity policy. + clusterModuleName := vmCtx.VM.Annotations[pkg.ClusterModuleNameKey] + if clusterModuleName == "" { + return nil + } + + // Find ClusterModule UUID from the ResourcePolicy. + _, moduleUUID := clustermodules.FindClusterModuleUUID(clusterModuleName, s.Cluster.Reference(), resourcePolicy) + if moduleUUID == "" { + return fmt.Errorf("ClusterModule %s not found", clusterModuleName) + } + + return s.Client.ClusterModuleClient().AddMoRefToModule(vmCtx, moduleUUID, resVM.MoRef()) +} + +func (s *Session) UpdateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + getUpdateArgsFn func() (*VMUpdateArgs, error)) (err error) { + + resVM := res.NewVMFromObject(vcVM) + + moVM, err := resVM.GetProperties(vmCtx, []string{"config", "runtime"}) + if err != nil { + return err + } + + defer func() { + updateErr := vmlifecycle.UpdateStatus(vmCtx, s.K8sClient, vcVM, nil) + if updateErr != nil { + vmCtx.Logger.Error(updateErr, "Updating VM status failed") + if err == nil { + err = updateErr + } + } + }() + + // Translate the VM's current power state into the VM Op power state value. + var existingPowerState vmopv1.VirtualMachinePowerState + switch moVM.Runtime.PowerState { + case vimTypes.VirtualMachinePowerStatePoweredOn: + existingPowerState = vmopv1.VirtualMachinePowerStateOn + case vimTypes.VirtualMachinePowerStatePoweredOff: + existingPowerState = vmopv1.VirtualMachinePowerStateOff + case vimTypes.VirtualMachinePowerStateSuspended: + existingPowerState = vmopv1.VirtualMachinePowerStateSuspended + } + + switch vmCtx.VM.Spec.PowerState { + case vmopv1.VirtualMachinePowerStateOff: + var powerOff bool + if existingPowerState == vmopv1.VirtualMachinePowerStateOn { + powerOff = true + } else if existingPowerState == vmopv1.VirtualMachinePowerStateSuspended { + if vmCtx.VM.Spec.PowerOffMode == vmopv1.VirtualMachinePowerOpModeHard { + powerOff = true + } + } + if powerOff { + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmCtx.VM.Spec.PowerOffMode) + } + + // BMV: We'll likely want to reconfigure a powered off VM too, but right now + // we'll defer that until the pre power on (and until more people complain + // that the UI appears wrong). + + case vmopv1.VirtualMachinePowerStateSuspended: + if existingPowerState == vmopv1.VirtualMachinePowerStateOn { + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmCtx.VM.Spec.SuspendMode) + } + + case vmopv1.VirtualMachinePowerStateOn: + config := moVM.Config + + // See GoVmomi's VirtualMachine::Device() explanation for this check. + if config == nil { + return fmt.Errorf("VM config is not available, connectionState=%s", moVM.Runtime.ConnectionState) + } + + switch existingPowerState { + case vmopv1.VirtualMachinePowerStateOn: + + // Check to see if a possible restart is required. + // Please note a VM may only be restarted if it is powered on. + if vmCtx.VM.Spec.NextRestartTime != "" { + + // If non-empty, the value of spec.nextRestartTime is guaranteed + // to be a valid RFC3339Nano timestamp due to the webhooks, + // however, we still check for the error due to testing that may + // not involve webhooks. + nextRestartTime, err := time.Parse( + time.RFC3339Nano, vmCtx.VM.Spec.NextRestartTime) + if err != nil { + return fmt.Errorf( + "spec.nextRestartTime %q cannot be parsed with %q %w", + vmCtx.VM.Spec.NextRestartTime, + time.RFC3339Nano, + err) + } + result, err := vmutil.RestartAndWait( + logr.NewContext(vmCtx, vmCtx.Logger), + vcVM.Client(), + vmutil.ManagedObjectFromObject(vcVM), + false, + nextRestartTime, + vmutil.ParsePowerOpMode(string(vmCtx.VM.Spec.RestartMode))) + if err != nil { + return err + } + if result.AnyChange() { + lastRestartTime := metav1.NewTime(nextRestartTime) + vmCtx.VM.Status.LastRestartTime = &lastRestartTime + } + } + + // Do not pass classConfigSpec to poweredOnVMReconfigure when VM is + // already powered on since we do not have to get VM class at this + // point. + return s.poweredOnVMReconfigure(vmCtx, resVM, config) + + case vmopv1.VirtualMachinePowerStateSuspended: + // A suspended VM cannot be reconfigured. + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmopv1.VirtualMachinePowerOpModeHard) + } + + updateArgs, err := getUpdateArgsFn() + if err != nil { + return err + } + + // TODO: Find a better place for this? + if err := s.attachClusterModule(vmCtx, resVM, updateArgs.ResourcePolicy); err != nil { + return err + } + + if err := s.prepareVMForPowerOn(vmCtx, resVM, config, updateArgs); err != nil { + return err + } + + if err := resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmopv1.VirtualMachinePowerOpModeHard); err != nil { + return err + } + + if vmCtx.VM.Annotations == nil { + vmCtx.VM.Annotations = map[string]string{} + } + vmCtx.VM.Annotations[FirstBootDoneAnnotation] = "true" + } + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go new file mode 100644 index 000000000..bd3098660 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go @@ -0,0 +1,1236 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var _ = Describe("Update ConfigSpec", func() { + + var ( + config *vimTypes.VirtualMachineConfigInfo + configSpec *vimTypes.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + config = &vimTypes.VirtualMachineConfigInfo{} + configSpec = &vimTypes.VirtualMachineConfigSpec{} + }) + + // Just a few examples for testing these things here. Need to think more about whether this + // is a good way or not. Probably better to do this via UpdateVirtualMachine when we have + // better integration tests. + + Context("Basic Hardware", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateHardwareConfigSpec(config, configSpec, vmClassSpec) + }) + + Context("Updates Hardware", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Cpus = 42 + vmClassSpec.Hardware.Memory = resource.MustParse("2000Mi") + }) + + It("config spec is not empty", func() { + Expect(configSpec.NumCPUs).To(BeNumerically("==", 42)) + Expect(configSpec.MemoryMB).To(BeNumerically("==", 2000)) + }) + }) + + Context("config already matches", func() { + BeforeEach(func() { + config.Hardware.NumCPU = 42 + vmClassSpec.Hardware.Cpus = int64(config.Hardware.NumCPU) + config.Hardware.MemoryMB = 1500 + vmClassSpec.Hardware.Memory = resource.MustParse(fmt.Sprintf("%dMi", config.Hardware.MemoryMB)) + }) + + It("config spec show no changes", func() { + Expect(configSpec.NumCPUs).To(BeZero()) + Expect(configSpec.MemoryMB).To(BeZero()) + }) + }) + }) + + Context("CPU Allocation", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + var minCPUFreq uint64 = 1 + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecCPUAllocation(config, configSpec, vmClassSpec, minCPUFreq) + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + + Context("config matches class policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Requests.Cpu = r + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + }) + + Context("config matches class policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Limits.Cpu = r + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + }) + + Context("config matches is different from policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(10 * virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Limits.Cpu = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.CpuAllocation.Reservation).To(BeNil()) + Expect(configSpec.CpuAllocation.Limit).ToNot(BeNil()) + Expect(*configSpec.CpuAllocation.Limit).To(BeNumerically("==", 100*1024*1024)) + }) + }) + + Context("config matches is different from policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(10 * virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Requests.Cpu = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.CpuAllocation.Limit).To(BeNil()) + Expect(configSpec.CpuAllocation.Reservation).ToNot(BeNil()) + Expect(*configSpec.CpuAllocation.Reservation).To(BeNumerically("==", 100*1024*1024)) + }) + }) + }) + + Context("Memory Allocation", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecMemoryAllocation(config, configSpec, vmClassSpec) + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + + Context("config matches class policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Requests.Memory = r + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + }) + + Context("config matches class policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Limits.Memory = r + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + }) + + Context("config matches is different from policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(10 * virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Limits.Memory = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation.Reservation).To(BeNil()) + Expect(configSpec.MemoryAllocation.Limit).ToNot(BeNil()) + Expect(*configSpec.MemoryAllocation.Limit).To(BeNumerically("==", 100)) + }) + }) + + Context("config matches is different from policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(10 * virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Requests.Memory = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation.Limit).To(BeNil()) + Expect(configSpec.MemoryAllocation.Reservation).ToNot(BeNil()) + Expect(*configSpec.MemoryAllocation.Reservation).To(BeNumerically("==", 100)) + }) + }) + }) + + Context("ExtraConfig", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + var vm *vmopv1.VirtualMachine + var globalExtraConfig map[string]string + var ecMap map[string]string + var imageV1Alpha1Compatible bool + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + globalExtraConfig = make(map[string]string) + classConfigSpec = nil + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecExtraConfig( + config, + configSpec, + classConfigSpec, + vmClassSpec, + vm, + globalExtraConfig, + imageV1Alpha1Compatible) + + ecMap = make(map[string]string) + for _, ec := range configSpec.ExtraConfig { + if optionValue := ec.GetOptionValue(); optionValue != nil { + ecMap[optionValue.Key] = optionValue.Value.(string) + } + } + }) + + Context("Empty input", func() { + It("No changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("Updates configSpec.ExtraConfig", func() { + BeforeEach(func() { + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{ + Key: constants.VMOperatorV1Alpha1ExtraConfigKey, Value: constants.VMOperatorV1Alpha1ConfigReady}) + globalExtraConfig["guestinfo.test"] = "test" + globalExtraConfig["global"] = "test" + imageV1Alpha1Compatible = true + }) + + It("Expected configSpec.ExtraConfig", func() { + By("VM Image compatible", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + + By("Global map", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.test", "test")) + Expect(ecMap).To(HaveKeyWithValue("global", "test")) + }) + }) + + Context("When VM uses metadata transport types other than CloudInit", func() { + BeforeEach(func() { + vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + }) + It("defer cloud-init extra config is enabled", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + }) + + Context("When VM uses CloudInit metadata transport type", func() { + BeforeEach(func() { + vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + }) + It("defer cloud-init extra config is not enabled", func() { + Expect(ecMap).ToNot(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + }) + }) + + Context("ExtraConfig value already exists", func() { + BeforeEach(func() { + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{Key: "foo", Value: "bar"}) + globalExtraConfig["foo"] = "bar" + }) + + It("No changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("InstanceStorage related tests", func() { + + Context("When InstanceStorage is NOT configured on VM", func() { + It("No Changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("When InstanceStorage is configured on VM", func() { + BeforeEach(func() { + vm.Spec.Volumes = append(vm.Spec.Volumes, vmopv1.VirtualMachineVolume{ + Name: "pvc-volume-1", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-volume-1", + }, + InstanceVolumeClaim: &vmopv1.InstanceVolumeClaimVolumeSource{ + StorageClass: "dummyStorageClass", + Size: resource.MustParse("256Gi"), + }, + }, + }, + }) + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + }) + }) + + Context("ThunderPciDevices related test", func() { + + Context("when virtual devices are not present", func() { + It("No Changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("when vGPU device is available", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Devices = vmopv1.VirtualDevices{VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "test-vgpu-profile", + }, + }} + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + + It("PCI passthru MMIO extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + + Context("when PCI passthru MMIO override annotation is set", func() { + BeforeEach(func() { + vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] = "12345" + }) + + It("PCI passthru MMIO extraConfig should be set to override annotation value", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, "12345")) + }) + }) + }) + + Context("when DDPIO device is available", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Devices = vmopv1.VirtualDevices{DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 123, + DeviceID: 24, + CustomLabel: "", + }, + }} + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + + It("PCI passthru MMIO extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + + Context("when PCI passthru MMIO override annotation is set", func() { + BeforeEach(func() { + vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] = "12345" + }) + + It("PCI passthru MMIO extraConfig should be set to override annotation value", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, "12345")) + }) + }) + }) + }) + + Context("when VM_Class_as_Config_DaynDate FSS is enabled", func() { + var oldVMClassAsConfigDaynDateFunc func() bool + const dummyKey = "dummy-key" + const dummyVal = "dummy-val" + + BeforeEach(func() { + oldVMClassAsConfigDaynDateFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigDaynDateFunc + }) + + Context("classConfigSpec extra config is not nil", func() { + BeforeEach(func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ExtraConfig: []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{ + Key: dummyKey + "-1", + Value: dummyVal + "-1", + }, + &vimTypes.OptionValue{ + Key: dummyKey + "-2", + Value: dummyVal + "-2", + }, + }, + } + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{Key: "hello", Value: "world"}) + }) + It("vm extra config overlaps with global extra config", func() { + globalExtraConfig["hello"] = "world" + + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-1", dummyVal+"-1")) + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-2", dummyVal+"-2")) + Expect(ecMap).ToNot(HaveKeyWithValue("hello", "world")) + }) + + It("global extra config overlaps with class config spec - class config spec takes precedence", func() { + globalExtraConfig[dummyKey+"-1"] = dummyVal + "-3" + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-1", dummyVal+"-1")) + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-2", dummyVal+"-2")) + }) + + Context("class config spec has vGPU and DDPIO devices", func() { + BeforeEach(func() { + classConfigSpec.DeviceChange = []vimTypes.BaseVirtualDeviceConfigSpec{ + &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "SampleLabel2", + }, + }, + }, + }, + } + + }) + + It("extraConfig Map has MMIO and MMPowerOff related keys added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + }) + }) + }) + }) + + Context("ChangeBlockTracking", func() { + var vmSpec vmopv1.VirtualMachineSpec + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + + BeforeEach(func() { + config.ChangeTrackingEnabled = nil + classConfigSpec = nil + }) + + AfterEach(func() { + configSpec.ChangeTrackingEnabled = nil + }) + + It("cbt and status cbt unset", func() { + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("configSpec cbt set to true OMG", func() { + config.ChangeTrackingEnabled = pointer.Bool(true) + vmSpec.Advanced.ChangeBlockTracking = false + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeFalse()) + }) + + It("configSpec cbt set to false", func() { + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + + It("configSpec cbt matches", func() { + config.ChangeTrackingEnabled = pointer.Bool(true) + vmSpec.Advanced.ChangeBlockTracking = true + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("classConfigSpec not nil and is ignored", func() { + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(false), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + + Context("VM_Class_as_Config_DaynDate FSS is enabled", func() { + var oldVMClassAsConfigDaynDateFunc func() bool + BeforeEach(func() { + oldVMClassAsConfigDaynDateFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigDaynDateFunc + }) + + It("classConfigSpec not nil and same as configInfo", func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(false), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("classConfigSpec not nil, different from configInfo, overrides vm spec cbt", func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(true), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + }) + }) + + Context("Firmware", func() { + var vm *vmopv1.VirtualMachine + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + config.Firmware = "bios" + }) + + It("No firmware annotation", func() { + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + + It("Set firmware annotation equal to current vm firmware", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = config.Firmware + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + + It("Set firmware annotation differing to current vm firmware", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "efi" + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(Equal("efi")) + }) + + It("Set firmware annotation to an invalid value", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "invalidfirmware" + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + }) + + Context("DeviceGroups", func() { + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + + BeforeEach(func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{} + }) + + It("No DeviceGroups set in class config spec", func() { + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).To(BeNil()) + }) + + It("DeviceGroups set in class config spec", func() { + classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(400), + }, + }, + } + + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).NotTo(BeNil()) + Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) + deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() + Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) + }) + + It("configInfo DeviceGroups set with vals different than the class config spec", func() { + classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(400), + }, + }, + } + + config.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(500), + }, + }, + } + + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).NotTo(BeNil()) + Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) + deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() + Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) + }) + }) + + Context("Ethernet Card Changes", func() { + var expectedList object.VirtualDeviceList + var currentList object.VirtualDeviceList + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + var dvpg1 *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo + var dvpg2 *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo + var err error + + BeforeEach(func() { + dvpg1 = &vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo{ + Port: vimTypes.DistributedVirtualSwitchPortConnection{ + PortgroupKey: "key1", + SwitchUuid: "uuid1", + }, + } + + dvpg2 = &vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo{ + Port: vimTypes.DistributedVirtualSwitchPortConnection{ + PortgroupKey: "key2", + SwitchUuid: "uuid2", + }, + } + }) + + JustBeforeEach(func() { + deviceChanges, err = session.UpdateEthCardDeviceChanges(expectedList, currentList) + }) + + AfterEach(func() { + currentList = nil + expectedList = nil + }) + + Context("No devices", func() { + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + + Context("No device change when nothing changes", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + It("returns no device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(0)) + }) + }) + + Context("Add device", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + }) + + It("returns add device change", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(1)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when backing change", func() { + var card1 vimTypes.BaseVirtualDevice + var card2 vimTypes.BaseVirtualDevice + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg2) + Expect(err).ToNot(HaveOccurred()) + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when MAC address is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = "mac1" + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = "mac2" + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("When WCP_VMClass_as_Config is enabled, Add and remove device when card type is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + var oldVMClassAsConfigFunc func() bool + + BeforeEach(func() { + oldVMClassAsConfigFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet2", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigFunc + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when ExternalID is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = "ext1" + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = "ext2" + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Keeps existing device with same backing", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + }) + + Context("Create vSphere PCI device", func() { + var vgpuDevices = []vmopv1.VGPUDevice{ + { + ProfileName: "SampleProfile", + }, + } + var ddpioDevices = []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 42, + DeviceID: 43, + CustomLabel: "SampleLabel", + }, + } + var pciDevices vmopv1.VirtualDevices + Context("For VM Class Spec vGPU device", func() { + BeforeEach(func() { + pciDevices = vmopv1.VirtualDevices{ + VGPUDevices: vgpuDevices, + } + }) + It("should create vSphere device with VmiopBackingInfo", func() { + vSphereDevices := virtualmachine.CreatePCIDevicesFromVMClass(pciDevices) + Expect(vSphereDevices).To(HaveLen(1)) + virtualDevice := vSphereDevices[0].GetVirtualDevice() + backing := virtualDevice.Backing.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo) + Expect(backing.Vgpu).To(Equal(pciDevices.VGPUDevices[0].ProfileName)) + }) + }) + Context("For VM Class Spec Dynamic DirectPath I/O device", func() { + BeforeEach(func() { + pciDevices = vmopv1.VirtualDevices{ + DynamicDirectPathIODevices: ddpioDevices, + } + }) + It("should create vSphere device with DynamicBackingInfo", func() { + vSphereDevices := virtualmachine.CreatePCIDevicesFromVMClass(pciDevices) + Expect(vSphereDevices).To(HaveLen(1)) + virtualDevice := vSphereDevices[0].GetVirtualDevice() + backing := virtualDevice.Backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo) + Expect(backing.AllowedDevice[0].DeviceId).To(Equal(int32(pciDevices.DynamicDirectPathIODevices[0].DeviceID))) + Expect(backing.AllowedDevice[0].VendorId).To(Equal(int32(pciDevices.DynamicDirectPathIODevices[0].VendorID))) + Expect(backing.CustomLabel).To(Equal(pciDevices.DynamicDirectPathIODevices[0].CustomLabel)) + }) + }) + + When("PCI devices from ConfigSpec are specified", func() { + + var devIn []*vimTypes.VirtualPCIPassthrough + + Context("For ConfigSpec VGPU device", func() { + BeforeEach(func() { + devIn = []*vimTypes.VirtualPCIPassthrough{ + { + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "configspec-profile", + }, + }, + }, + } + }) + It("should create vSphere device with VmiopBackingInfo", func() { + devList := virtualmachine.CreatePCIDevicesFromConfigSpec(devIn) + Expect(devList).To(HaveLen(1)) + + Expect(devList[0]).ToNot(BeNil()) + Expect(devList[0]).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthrough{})) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).ToNot(BeNil()) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthroughVmiopBackingInfo{})) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo).Vgpu).To(Equal("configspec-profile")) + }) + }) + + Context("For ConfigSpec DirectPath I/O device", func() { + BeforeEach(func() { + devIn = []*vimTypes.VirtualPCIPassthrough{ + { + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + CustomLabel: "configspec-ddpio-label", + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 456, + DeviceId: 457, + }, + }, + }, + }, + }, + } + }) + It("should create vSphere device with DynamicBackingInfo", func() { + devList := virtualmachine.CreatePCIDevicesFromConfigSpec(devIn) + Expect(devList).To(HaveLen(1)) + + Expect(devList[0]).ToNot(BeNil()) + Expect(devList[0]).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthrough{})) + + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).ToNot(BeNil()) + backing := devList[0].(*vimTypes.VirtualPCIPassthrough).Backing + Expect(backing).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthroughDynamicBackingInfo{})) + + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).CustomLabel).To(Equal("configspec-ddpio-label")) + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).AllowedDevice[0].VendorId).To(BeEquivalentTo(456)) + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).AllowedDevice[0].DeviceId).To(BeEquivalentTo(457)) + }) + }) + }) + }) + + Context("PCI Device Changes", func() { + var ( + currentList, expectedList object.VirtualDeviceList + deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + err error + + // Variables related to vGPU devices. + backingInfo1, backingInfo2 *vimTypes.VirtualPCIPassthroughVmiopBackingInfo + deviceKey1, deviceKey2 int32 + vGPUDevice1, vGPUDevice2 vimTypes.BaseVirtualDevice + + // Variables related to dynamicDirectPathIO devices. + allowedDev1, allowedDev2 vimTypes.VirtualPCIPassthroughAllowedDevice + backingInfo3, backingInfo4 *vimTypes.VirtualPCIPassthroughDynamicBackingInfo + deviceKey3, deviceKey4 int32 + dynamicDirectPathIODev1, dynamicDirectPathIODev2 vimTypes.BaseVirtualDevice + ) + + BeforeEach(func() { + backingInfo1 = &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{Vgpu: "mockup-vmiop1"} + backingInfo2 = &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{Vgpu: "mockup-vmiop2"} + deviceKey1 = int32(-200) + deviceKey2 = int32(-201) + vGPUDevice1 = virtualmachine.CreatePCIPassThroughDevice(deviceKey1, backingInfo1) + vGPUDevice2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey2, backingInfo2) + + allowedDev1 = vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: 1000, + DeviceId: 100, + } + allowedDev2 = vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: 2000, + DeviceId: 200, + } + backingInfo3 = &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev1}, + CustomLabel: "sampleLabel3", + } + backingInfo4 = &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev2}, + CustomLabel: "sampleLabel4", + } + deviceKey3 = int32(-202) + deviceKey4 = int32(-203) + dynamicDirectPathIODev1 = virtualmachine.CreatePCIPassThroughDevice(deviceKey3, backingInfo3) + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfo4) + }) + + JustBeforeEach(func() { + deviceChanges, err = session.UpdatePCIDeviceChanges(expectedList, currentList) + }) + + AfterEach(func() { + currentList = nil + expectedList = nil + }) + + Context("No devices", func() { + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + + Context("Adding vGPU and dynamicDirectPathIO devices with different backing info", func() { + BeforeEach(func() { + expectedList = append(expectedList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice2) + expectedList = append(expectedList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(len(expectedList))) + + for idx, dev := range deviceChanges { + configSpec := dev.GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[idx].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("Adding vGPU and dynamicDirectPathIO devices with same backing info", func() { + BeforeEach(func() { + expectedList = append(expectedList, vGPUDevice1) + // Creating a vGPUDevice with same backingInfo1 but different deviceKey. + vGPUDevice2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey2, backingInfo1) + expectedList = append(expectedList, vGPUDevice2) + expectedList = append(expectedList, dynamicDirectPathIODev1) + // Creating a dynamicDirectPathIO device with same backingInfo3 but different deviceKey. + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfo3) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(len(expectedList))) + + for idx, dev := range deviceChanges { + configSpec := dev.GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[idx].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("When the expected and current lists have DDPIO devices with different custom labels", func() { + BeforeEach(func() { + expectedList = []vimTypes.BaseVirtualDevice{dynamicDirectPathIODev1} + // Creating a dynamicDirectPathIO device with same backing info except for the custom label. + backingInfoDiffCustomLabel := &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: backingInfo3.AllowedDevice, + CustomLabel: "DifferentLabel", + } + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfoDiffCustomLabel) + currentList = []vimTypes.BaseVirtualDevice{dynamicDirectPathIODev2} + }) + + It("should return add and remove device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(currentList[0].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[0].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("When the expected and current list of pciDevices have different Devices", func() { + BeforeEach(func() { + currentList = append(currentList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice2) + currentList = append(currentList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add and remove device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(4)) + + for i := 0; i < 2; i++ { + configSpec := deviceChanges[i].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(currentList[i].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + } + + for i := 2; i < 4; i++ { + configSpec := deviceChanges[i].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[i-2].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("When the expected and current list of pciDevices have same Devices", func() { + BeforeEach(func() { + currentList = append(currentList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice1) + currentList = append(currentList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev1) + }) + + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/storage/provisioning.go b/pkg/vmprovider/providers/vsphere2/storage/provisioning.go new file mode 100644 index 000000000..5a3997a65 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/storage/provisioning.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "github.com/pkg/errors" + "github.com/vmware/govmomi/pbm" + pbmTypes "github.com/vmware/govmomi/pbm/types" + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" +) + +// getProfileProportionalCapacity returns the storage profile "proportionalCapacity" value if the +// policy is for vSAN. The proportionalCapacity is percentage of the logical size of the storage +// object that will be reserved upon provisioning. Returns -1 if not specified. +// The UI presents options for "thin" (0%), 25%, 50%, 75% and "thick" (100%). +func getProfileProportionalCapacity(profile pbmTypes.BasePbmProfile) int32 { + capProfile, ok := profile.(*pbmTypes.PbmCapabilityProfile) + if !ok { + return -1 + } + + if capProfile.ResourceType.ResourceType != string(pbmTypes.PbmProfileResourceTypeEnumSTORAGE) { + return -1 + } + + if capProfile.ProfileCategory != string(pbmTypes.PbmProfileCategoryEnumREQUIREMENT) { + return -1 + } + + sub, ok := capProfile.Constraints.(*pbmTypes.PbmCapabilitySubProfileConstraints) + if !ok { + return -1 + } + + for _, p := range sub.SubProfiles { + for _, capability := range p.Capability { + if capability.Id.Namespace != "VSAN" || capability.Id.Id != "proportionalCapacity" { + continue + } + + for _, c := range capability.Constraint { + for _, prop := range c.PropertyInstance { + if prop.Id != capability.Id.Id { + continue + } + if val, ok := prop.Value.(int32); ok { + return val + } + } + } + } + } + + return -1 +} + +// GetDiskProvisioningForProfile returns the provisioning type for the storage profile if it has +// one specified. +func GetDiskProvisioningForProfile( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + storageProfileID string) (string, error) { + + c, err := pbm.NewClient(vmCtx, vcClient.VimClient()) + if err != nil { + return "", err + } + + profiles, err := c.RetrieveContent(vmCtx, []pbmTypes.PbmProfileId{{UniqueId: storageProfileID}}) + if err != nil { + return "", errors.Wrapf(err, "Failed to get storage profiles for ID: %s", storageProfileID) + } + + for _, p := range profiles { + switch getProfileProportionalCapacity(p) { + case 0: + return string(vimTypes.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil + case 100: + return string(vimTypes.OvfCreateImportSpecParamsDiskProvisioningTypeThick), nil + } + } + + return "", nil +} diff --git a/pkg/vmprovider/providers/vsphere2/storage/storageclass.go b/pkg/vmprovider/providers/vsphere2/storage/storageclass.go new file mode 100644 index 000000000..511e651a0 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/storage/storageclass.go @@ -0,0 +1,83 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + + storagev1 "k8s.io/api/storage/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +// GetStoragePolicyID returns Storage Policy ID from Storage Class Name. +func GetStoragePolicyID( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + storageClassName string) (string, error) { + + sc := &storagev1.StorageClass{} + if err := client.Get(vmCtx, ctrlclient.ObjectKey{Name: storageClassName}, sc); err != nil { + vmCtx.Logger.Error(err, "Failed to get StorageClass", "storageClass", storageClassName) + return "", err + } + + policyID, ok := sc.Parameters["storagePolicyID"] + if !ok { + return "", fmt.Errorf("StorageClass %s does not have 'storagePolicyID' parameter", storageClassName) + } + + return policyID, nil +} + +// GetVMStoragePoliciesIDs returns a map of storage class names to their storage policy IDs. +func GetVMStoragePoliciesIDs( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client) (map[string]string, error) { + + storageClassNames := getVMStorageClassNames(vmCtx.VM) + storageClassesToIDs := map[string]string{} + + for _, name := range storageClassNames { + if _, ok := storageClassesToIDs[name]; !ok { + id, err := GetStoragePolicyID(vmCtx, client, name) + if err != nil { + return nil, err + } + + storageClassesToIDs[name] = id + } + } + + return storageClassesToIDs, nil +} + +func getVMStorageClassNames(vm *vmopv1.VirtualMachine) []string { + var names []string + + if vm.Spec.StorageClass != "" { + names = append(names, vm.Spec.StorageClass) + } + + for _, vol := range vm.Spec.Volumes { + var storageClass string + + claim := vol.PersistentVolumeClaim + if claim != nil { + if isClaim := claim.InstanceVolumeClaim; isClaim != nil { + storageClass = isClaim.StorageClass + } else { //nolint + // TODO: Fetch claim.ClaimName PVC to get the StorageClass. + } + } + + if storageClass != "" { + names = append(names, storageClass) + } + } + + return names +} diff --git a/pkg/vmprovider/providers/vsphere2/test/pki.go b/pkg/vmprovider/providers/vsphere2/test/pki.go new file mode 100644 index 000000000..557569440 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/pki.go @@ -0,0 +1,70 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "time" + + . "github.com/onsi/gomega" +) + +func GeneratePrivateKey() *rsa.PrivateKey { + reader := rand.Reader + bitSize := 2048 + + // Based on https://golang.org/src/crypto/tls/generate_cert.go + privateKey, err := rsa.GenerateKey(reader, bitSize) + if err != nil { + panic("failed to generate private key") + } + return privateKey +} + +func GenerateSelfSignedCert() (string, string) { + priv := GeneratePrivateKey() + now := time.Now() + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + Expect(err).NotTo(HaveOccurred()) + + template := x509.Certificate{ + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + }, + SerialNumber: serialNumber, + NotBefore: now, + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + template.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + Expect(err).NotTo(HaveOccurred()) + certOut, err := os.CreateTemp("", "cert.pem") + Expect(err).NotTo(HaveOccurred()) + err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + Expect(err).NotTo(HaveOccurred()) + err = certOut.Close() + Expect(err).NotTo(HaveOccurred()) + + keyOut, err := os.CreateTemp("", "key.pem") + Expect(err).NotTo(HaveOccurred()) + privBytes, err := x509.MarshalPKCS8PrivateKey(priv) + Expect(err).NotTo(HaveOccurred()) + err = pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + Expect(err).NotTo(HaveOccurred()) + err = keyOut.Close() + Expect(err).NotTo(HaveOccurred()) + + return keyOut.Name(), certOut.Name() +} diff --git a/pkg/vmprovider/providers/vsphere2/test/suite.go b/pkg/vmprovider/providers/vsphere2/test/suite.go new file mode 100644 index 000000000..cf496c076 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/suite.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "context" + "crypto/tls" + "os" + + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + _ "github.com/vmware/govmomi/vapi/simulator" // blank import the VAPI simulator bindings +) + +func BeforeSuite() (ctx context.Context, + model *simulator.Model, + server *simulator.Server, + tlsKeyPath, tlsCertPath string, + tlsModel *simulator.Model, + tlsServer *simulator.Server) { + + ctx = context.Background() + + // Set up a simulator for testing most client interactions (ignoring TLS) + model, server = SetupModelAndServerWithSettings(&tls.Config{ + MinVersion: tls.VersionTLS12, + }) + + // Set up a second simulator for testing TLS. + tlsKeyPath, tlsCertPath = GenerateSelfSignedCert() + tlsCert, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) + Expect(err).NotTo(HaveOccurred()) + tlsModel, tlsServer = SetupModelAndServerWithSettings(&tls.Config{ + Certificates: []tls.Certificate{ + tlsCert, + }, + PreferServerCipherSuites: true, + MinVersion: tls.VersionTLS12, + }) + + return +} + +func AfterSuite( + ctx context.Context, + model *simulator.Model, + server *simulator.Server, + tlsKeyPath, tlsCertPath string, + tlsModel *simulator.Model, + tlsServer *simulator.Server) { + + server.Close() + model.Remove() + + tlsServer.Close() + tlsModel.Remove() + + _ = os.Remove(tlsKeyPath) + _ = os.Remove(tlsCertPath) +} diff --git a/pkg/vmprovider/providers/vsphere2/test/vcsim.go b/pkg/vmprovider/providers/vsphere2/test/vcsim.go new file mode 100644 index 000000000..607ed81e2 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/vcsim.go @@ -0,0 +1,31 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "crypto/tls" + + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" +) + +func SetupModelAndServerWithSettings(tlsConfig *tls.Config) (*simulator.Model, *simulator.Server) { + newModel := simulator.VPX() + + // By Default, the Model being used by vcsim has two ResourcePools + // (one for the cluster and host each). Setting Model.Host=0 ensures + // we only have one ResourcePool, making it easier to pick the + // ResourcePool without having to look up using a hardcoded path. + newModel.Host = 0 + + err := newModel.Create() + Expect(err).ToNot(HaveOccurred()) + + newModel.Service.RegisterEndpoints = true + + newModel.Service.TLS = tlsConfig + newServer := newModel.Service.NewServer() + + return newModel, newServer +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go b/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go new file mode 100644 index 000000000..2cbee7adb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25/mo" +) + +// ClusterMinCPUFreq returns the minimum frequency across all the hosts in the cluster. This is needed to +// convert the CPU requirements specified in cores to MHz. vSphere core is assumed to be equivalent to the +// value of min frequency. This function is adapted from wcp schedext. +func ClusterMinCPUFreq(ctx goctx.Context, cluster *object.ClusterComputeResource) (uint64, error) { + var cr mo.ComputeResource + if err := cluster.Properties(ctx, cluster.Reference(), []string{"host"}, &cr); err != nil { + return 0, err + } + + if len(cr.Host) == 0 { + return 0, nil + } + + var hosts []mo.HostSystem + pc := property.DefaultCollector(cluster.Client()) + if err := pc.Retrieve(ctx, cr.Host, []string{"summary"}, &hosts); err != nil { + return 0, err + } + + var minFreq uint64 + for _, h := range hosts { + if hw := h.Summary.Hardware; hw != nil { + hostCPUMHz := uint64(hw.CpuMhz) + if hostCPUMHz < minFreq || minFreq == 0 { + minFreq = hostCPUMHz + } + } + } + + return minFreq, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go new file mode 100644 index 000000000..5b90b5145 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func clusterTests() { + Describe("ClusterMinCPUFreq", minFreq) +} + +func minFreq() { + // Hardcoded value in govmomi simulator/esx/host_system.go + const expectedCPUFreq = 2294 + + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("ClusterMinCPUFreq", func() { + It("returns min freq of hosts in cluster", func() { + cpuFreq, err := vcenter.ClusterMinCPUFreq(ctx, ctx.GetSingleClusterCompute()) + Expect(err).ToNot(HaveOccurred()) + Expect(cpuFreq).Should(BeEquivalentTo(expectedCPUFreq)) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/folder.go b/pkg/vmprovider/providers/vsphere2/vcenter/folder.go new file mode 100644 index 000000000..414a08a67 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/folder.go @@ -0,0 +1,138 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" +) + +// GetFolderByMoID returns the vim Folder for the MoID. +func GetFolderByMoID( + ctx goctx.Context, + finder *find.Finder, + folderMoID string) (*object.Folder, error) { + + o, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "Folder", Value: folderMoID}) + if err != nil { + return nil, err + } + + return o.(*object.Folder), nil +} + +// GetChildFolder gets the named child Folder from the parent Folder. +func GetChildFolder( + ctx goctx.Context, + parentFolder *object.Folder, + childName string) (*object.Folder, error) { + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return nil, err + } else if childFolder == nil { + return nil, fmt.Errorf("folder child %s not found under parent Folder %s", + childName, parentFolder.Reference().Value) + } + + return childFolder, nil +} + +// DoesChildFolderExist returns if the named child Folder exists under the parent Folder. +func DoesChildFolderExist( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) (bool, error) { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return false, err + } + + return childFolder != nil, nil +} + +// CreateFolder creates the named child Folder under the parent Folder. +func CreateFolder( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) (string, error) { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return "", err + } + + if childFolder == nil { + folder, err := parentFolder.CreateFolder(ctx, childName) + if err != nil { + return "", err + } + + childFolder = folder + } + + return childFolder.Reference().Value, nil +} + +// DeleteChildFolder deletes the child Folder under the parent Folder. +func DeleteChildFolder( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) error { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil || childFolder == nil { + return err + } + + task, err := childFolder.Destroy(ctx) + if err != nil { + return err + } + + if taskResult, err := task.WaitForResult(ctx); err != nil { + if taskResult == nil || taskResult.Error == nil { + return err + } + return fmt.Errorf("destroy Folder %s task failed: %w: %s", + childFolder.Reference().Value, err, taskResult.Error.LocalizedMessage) + } + + return nil +} + +func findChildFolder( + ctx goctx.Context, + parentFolder *object.Folder, + childName string) (*object.Folder, error) { + + objRef, err := object.NewSearchIndex(parentFolder.Client()).FindChild(ctx, parentFolder.Reference(), childName) + if err != nil { + return nil, err + } else if objRef == nil { + return nil, nil + } + + folder, ok := objRef.(*object.Folder) + if !ok { + return nil, fmt.Errorf("Folder child %q is not Folder but a %T", childName, objRef) //nolint + } + + return folder, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go new file mode 100644 index 000000000..99cfa9426 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go @@ -0,0 +1,173 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func folderTests() { + Describe("GetFolderByMoID", getFolderByMoID) + Describe("CreateDeleteExistsFolder", createDeleteExistsFolder) +} + +func getFolderByMoID() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("returns success", func() { + moID := nsInfo.Folder.Reference().Value + + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, moID) + Expect(err).ToNot(HaveOccurred()) + Expect(folder).ToNot(BeNil()) + Expect(folder.Name()).To(Equal(nsInfo.Namespace)) + }) + + It("returns error when moID does not exist", func() { + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, "bogus") + Expect(err).To(HaveOccurred()) + Expect(folder).To(BeNil()) + }) +} + +func createDeleteExistsFolder() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + + parentFolderMoID string + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + parentFolderMoID = nsInfo.Folder.Reference().Value + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + parentFolderMoID = "" + }) + + Context("CreateFolder", func() { + It("creates child Folder", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(childMoID).ToNot(BeEmpty()) + + By("NoOp when child Folder already exists", func() { + moID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(moID).To(Equal(childMoID)) + }) + + By("child Folder is found by MoID", func() { + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, childMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(folder.Reference().Value).To(Equal(childMoID)) + }) + }) + + It("returns error when parent Folder MoID does not exist", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, "bogus", "myFolder") + Expect(err).To(HaveOccurred()) + Expect(childMoID).To(BeEmpty()) + }) + }) + + Context("GetChildFolder", func() { + It("returns success when child Folder exists", func() { + childFolderMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + parentFolder := object.NewFolder(ctx.VCClient.Client, types.ManagedObjectReference{ + Type: "Folder", + Value: parentFolderMoID, + }) + + childFolder, err := vcenter.GetChildFolder(ctx, parentFolder, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(childFolder).ToNot(BeNil()) + Expect(childFolder.Reference().Value).To(Equal(childFolderMoID)) + }) + + It("returns error when child Folder does not exists", func() { + parentFolder := object.NewFolder(ctx.VCClient.Client, types.ManagedObjectReference{ + Type: "Folder", + Value: parentFolderMoID, + }) + + childFolder, err := vcenter.GetChildFolder(ctx, parentFolder, "myFolder") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found under parent Folder")) + Expect(childFolder).To(BeNil()) + }) + }) + + Context("DoesChildFolderExist", func() { + It("returns true when child Folder exists", func() { + _, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("returns false when child Folder does not exist", func() { + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + It("returns error when parent Folder MoID does not exist", func() { + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, "bogus", "myFolder") + Expect(err).To(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("DeleteFolder", func() { + It("deletes child Folder", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + err = vcenter.DeleteChildFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + By("child Folder is not found by MoID", func() { + _, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, childMoID) + Expect(err).To(HaveOccurred()) + }) + + By("NoOp when child does not exist", func() { + err := vcenter.DeleteChildFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go b/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go new file mode 100644 index 000000000..f6363a31c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go @@ -0,0 +1,152 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" +) + +// GetVirtualMachine gets the VM from VC, either by the MoID, UUID, or the inventory path. +func GetVirtualMachine( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vimClient *vim25.Client, + datacenter *object.Datacenter, + finder *find.Finder) (*object.VirtualMachine, error) { + + if uniqueID := vmCtx.VM.Status.UniqueID; uniqueID != "" { + if vm, err := findVMByMoID(vmCtx, finder, uniqueID); err == nil { + return vm, nil + } + } + + // For when we start to use the k8s VM.UID for the VC VM's InstanceUUID or UUID (aka BiosUUID): + /* + if instanceUUID := vmCtx.VM.UID; instanceUUID != "" { + if vm, err := findVMByUUID(vmCtx, vimClient, datacenter, string(instanceUUID), true); err == nil { + return vm, nil + } + } + */ + + return findVMByInventory(vmCtx, k8sClient, vimClient, finder) +} + +func findVMByMoID( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + moID string) (*object.VirtualMachine, error) { + + ref, err := finder.ObjectReference(vmCtx, types.ManagedObjectReference{Type: "VirtualMachine", Value: moID}) + if err != nil { + return nil, err + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VM but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via MoID", "path", vm.InventoryPath, "moID", moID) + return vm, nil +} + +//nolint:unused +func findVMByUUID( + vmCtx context.VirtualMachineContextA2, + vimClient *vim25.Client, + datacenter *object.Datacenter, + uuid string, + isInstanceUUID bool) (*object.VirtualMachine, error) { + + ref, err := object.NewSearchIndex(vimClient).FindByUuid(vmCtx, datacenter, uuid, true, &isInstanceUUID) + if err != nil { + return nil, fmt.Errorf("error finding object by UUID %q: %w", uuid, err) + } else if ref == nil { + return nil, fmt.Errorf("no VM found for UUID %q (instanceUUID: %v)", uuid, isInstanceUUID) + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VirtualMachine but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via UUID", "uuid", uuid, "isInstanceUUID", isInstanceUUID) + return vm, nil +} + +func findVMByInventory( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vimClient *vim25.Client, + finder *find.Finder) (*object.VirtualMachine, error) { + + // Note that we'll usually only get here to find the VM via its inventory path when we're first + // creating the VM. To determine the path, we need the NS Folder MoID and the VM's ResourcePolicy, + // if set, and we'll fetch these again as a part of createVirtualMachine(). For now, just re-fetch + // but we could pass the Folder MoID and ResourcePolicy to save a bit of duplicated work. + + folderMoID, err := topology.GetNamespaceFolderMoID(vmCtx, k8sClient, vmCtx.VM.Namespace) + if err != nil { + return nil, err + } + + // While we strictly only need the Folder's ManagedObjectReference below, use the Finder + // here to check if it actually exists. + folder, err := GetFolderByMoID(vmCtx, finder, folderMoID) + if err != nil { + return nil, fmt.Errorf("failed to get namespace Folder: %w", err) + } + + // When the VM has a ResourcePolicy, the VM is placed in a child folder under the namespace's folder. + if policyName := vmCtx.VM.Spec.Reserved.ResourcePolicyName; policyName != "" { + resourcePolicy := &vmopv1.VirtualMachineSetResourcePolicy{} + + key := ctrlclient.ObjectKey{Name: policyName, Namespace: vmCtx.VM.Namespace} + if err := k8sClient.Get(vmCtx, key, resourcePolicy); err != nil { + // Note that if VM does not exist, and we're about to create it, the ResourcePolicy is Get() + // again so the corresponding condition is almost always true if we don't hit an error here. + // Creating the VM with an explicit InstanceUUID is the easiest way out to avoid this. + return nil, fmt.Errorf("failed to get VirtualMachineSetResourcePolicy: %w", err) + } + + if folderName := resourcePolicy.Spec.Folder; folderName != "" { + childFolder, err := GetChildFolder(vmCtx, folder, folderName) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VirtualMachineSetResourcePolicy child Folder", + "parentPath", folder.InventoryPath, "folderName", folderName, "policyName", policyName) + return nil, err + } + + folder = childFolder + } + } + + ref, err := object.NewSearchIndex(vimClient).FindChild(vmCtx, folder.Reference(), vmCtx.VM.Name) + if err != nil { + return nil, err + } else if ref == nil { + // VM does not exist. + return nil, nil + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VM but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via inventory", + "parentFolderMoID", folder.Reference().Value, "moID", vm.Reference().Value) + return vm, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go new file mode 100644 index 000000000..29b48be5a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go @@ -0,0 +1,158 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/apimachinery/pkg/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func getVMTests() { + Describe("GetVirtualMachine", getVM) +} + +func getVM() { + // Use a VM that vcsim creates for us. + const vcVMName = "DC0_C0_RP0_VM0" + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + + vm := builder.DummyVirtualMachineA2() + vm.Name = "getvm-test" + vm.Namespace = nsInfo.Namespace + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + Context("Gets VM by inventory", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + task, err := vm.Clone(ctx, nsInfo.Folder, vmCtx.VM.Name, vimtypes.VirtualMachineCloneSpec{}) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + + It("returns nil if VM does not exist", func() { + vmCtx.VM.Name = "bogus" + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).To(BeNil()) + }) + + Context("Namespace Folder does not exist", func() { + BeforeEach(func() { + task, err := nsInfo.Folder.Destroy(vmCtx) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(vmCtx)).To(Succeed()) + }) + + It("returns error", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to get namespace Folder")) + Expect(vm).To(BeNil()) + }) + }) + + It("returns success when MoID is invalid", func() { + // Expect fallback to inventory. + vmCtx.VM.Status.UniqueID = "vm-bogus" + + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + }) + + Context("Gets VM when MoID is set", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + vmCtx.VM.Status.UniqueID = vm.Reference().Value + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + Expect(vm.Reference().Value).To(Equal(vmCtx.VM.Status.UniqueID)) + }) + }) + + // Not until we start setting either the InstanceUUID or BiosUUID + XContext("Gets VM by UUID", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vm.Properties(ctx, vm.Reference(), nil, &o)).To(Succeed()) + vmCtx.VM.UID = types.UID(o.Config.InstanceUuid) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + }) + + Context("Gets VM with ResourcePolicy by inventory", func() { + BeforeEach(func() { + resourcePolicy, folder := ctx.CreateVirtualMachineSetResourcePolicyA2("getvm-test", nsInfo) + vmCtx.VM.Spec.Reserved.ResourcePolicyName = resourcePolicy.Name + + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + task, err := vm.Clone(ctx, folder, vmCtx.VM.Name, vimtypes.VirtualMachineCloneSpec{}) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + + It("returns error when ResourcePolicy does not exist", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "bogus" + + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to get VirtualMachineSetResourcePolicy")) + Expect(vm).To(BeNil()) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/host.go b/pkg/vmprovider/providers/vsphere2/vcenter/host.go new file mode 100644 index 000000000..e121a5a45 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/host.go @@ -0,0 +1,41 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "fmt" + "strings" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" +) + +// GetESXHostFQDN returns the ESX host's FQDN. +func GetESXHostFQDN( + ctx context.Context, + vimClient *vim25.Client, + hostMoID string) (string, error) { + + hostMoRef := types.ManagedObjectReference{Type: "HostSystem", Value: hostMoID} + networkSys, err := object.NewHostSystem(vimClient, hostMoRef).ConfigManager().NetworkSystem(ctx) + if err != nil { + return "", fmt.Errorf("failed to get HostNetworkSystem for hostMoID %s: %w", hostMoID, err) + } + + var hostNetworkSys mo.HostNetworkSystem + if err := networkSys.Properties(ctx, networkSys.Reference(), []string{"dnsConfig"}, &hostNetworkSys); err != nil { + return "", fmt.Errorf("failed to get HostMoID %s DNSConfig prop: %w", hostMoID, err) + } + + if hostNetworkSys.DnsConfig == nil { + return "", fmt.Errorf("hostMoID %s HostNetworkSystem does not have DNSConfig", hostMoID) + } + + hostDNSConfig := hostNetworkSys.DnsConfig.GetHostDnsConfig() + hostFQDN := strings.TrimSuffix(hostDNSConfig.HostName+"."+hostDNSConfig.DomainName, ".") + return strings.ToLower(hostFQDN), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go new file mode 100644 index 000000000..9b7fbabdc --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func hostTests() { + Describe("GetESXHostFQDN", hostFQDN) +} + +func hostFQDN() { + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + + hostMoID string + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + testConfig.WithInstanceStorage = true + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + hosts, err := ctx.Finder.HostSystemList(ctx, "*") + Expect(err).ToNot(HaveOccurred()) + Expect(hosts).ToNot(BeEmpty()) + hostMoID = hosts[0].Reference().Value + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("GetESXHostFQDN", func() { + When("host does not have DNSConfig", func() { + BeforeEach(func() { + testConfig.WithInstanceStorage = false + }) + + It("returns expected error", func() { + _, err := vcenter.GetESXHostFQDN(ctx, ctx.VCClient.Client, hostMoID) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(" does not have DNSConfig")) + }) + }) + + It("returns expected host name for host", func() { + hostName, err := vcenter.GetESXHostFQDN(ctx, ctx.VCClient.Client, hostMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(hostName).Should(Equal(fmt.Sprintf("%s.vmop.vmware.com", hostMoID))) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go new file mode 100644 index 000000000..080f1c5af --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go @@ -0,0 +1,163 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +// GetResourcePoolByMoID returns the ResourcePool for the MoID. +func GetResourcePoolByMoID( + ctx goctx.Context, + finder *find.Finder, + rpMoID string) (*object.ResourcePool, error) { + + o, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + if err != nil { + return nil, err + } + + return o.(*object.ResourcePool), nil +} + +// GetResourcePoolOwnerMoRef returns the ClusterComputeResource MoID that owns the ResourcePool. +func GetResourcePoolOwnerMoRef( + ctx goctx.Context, + vimClient *vim25.Client, + rpMoID string) (types.ManagedObjectReference, error) { + + rp := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + objRef, err := rp.Owner(ctx) + if err != nil { + return types.ManagedObjectReference{}, err + } + + return objRef.Reference(), nil +} + +// GetChildResourcePool gets the named child ResourcePool from the parent ResourcePool. +func GetChildResourcePool( + ctx goctx.Context, + parentRP *object.ResourcePool, + childName string) (*object.ResourcePool, error) { + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil { + return nil, err + } else if childRP == nil { + return nil, fmt.Errorf("ResourcePool child %q not found under parent ResourcePool %s", + childName, parentRP.Reference().Value) + } + + return childRP, nil +} + +// DoesChildResourcePoolExist returns if the named child ResourcePool exists under the parent ResourcePool. +func DoesChildResourcePoolExist( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID, childName string) (bool, error) { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil { + return false, err + } + + return childRP != nil, nil +} + +// CreateOrUpdateChildResourcePool creates or updates the child ResourcePool under the parent ResourcePool. +func CreateOrUpdateChildResourcePool( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID string, + rpSpec *vmopv1.ResourcePoolSpec) (string, error) { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, rpSpec.Name) + if err != nil { + return "", err + } + + spec := types.DefaultResourceConfigSpec() // TODO Set reservations & limits from rpSpec + + if childRP == nil { + rp, err := parentRP.Create(ctx, rpSpec.Name, spec) + if err != nil { + return "", err + } + + childRP = rp + } else { //nolint + // TODO: // Finish this clause + } + + return childRP.Reference().Value, nil +} + +// DeleteChildResourcePool deletes the child ResourcePool under the parent ResourcePool. +func DeleteChildResourcePool( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID, childName string) error { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil || childRP == nil { + return err + } + + task, err := childRP.Destroy(ctx) + if err != nil { + return err + } + + if taskResult, err := task.WaitForResult(ctx); err != nil { + if taskResult == nil || taskResult.Error == nil { + return err + } + return fmt.Errorf("destroy ResourcePool %s task failed: %w: %s", + childRP.Reference().Value, err, taskResult.Error.LocalizedMessage) + } + + return nil +} + +func findChildRP( + ctx goctx.Context, + parentRP *object.ResourcePool, + childName string) (*object.ResourcePool, error) { + + objRef, err := object.NewSearchIndex(parentRP.Client()).FindChild(ctx, parentRP, childName) + if err != nil { + return nil, err + } else if objRef == nil { + // FindChild() returns nil when child name is not found. + return nil, nil + } + + childRP, ok := objRef.(*object.ResourcePool) + if !ok { + return nil, fmt.Errorf("ResourcePool child %q is not a ResourcePool but a %T", childName, objRef) + } + + return childRP, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go new file mode 100644 index 000000000..e0e16208d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go @@ -0,0 +1,200 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func resourcePoolTests() { + Describe("GetResourcePool", getResourcePoolTests) + Describe("CreateDeleteExistResourcePoolChild", createDeleteExistResourcePoolChild) +} + +func getResourcePoolTests() { + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + nsRP *object.ResourcePool + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + nsRP = ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + nsRP = nil + }) + + Context("GetResourcePoolByMoID", func() { + It("returns success", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, nsRP.Reference().Value) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).ToNot(BeNil()) + Expect(rp.Reference()).To(Equal(nsRP.Reference())) + }) + + It("returns error when MoID does not exist", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, "bogus") + Expect(err).To(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + }) + + Context("GetResourcePoolOwnerMoRef", func() { + It("returns success", func() { + ccr, err := vcenter.GetResourcePoolOwnerMoRef(ctx, ctx.VCClient.Client, nsRP.Reference().Value) + Expect(err).ToNot(HaveOccurred()) + Expect(ccr).To(Equal(ctx.GetSingleClusterCompute().Reference())) + }) + + It("returns error when MoID does not exist", func() { + _, err := vcenter.GetResourcePoolOwnerMoRef(ctx, ctx.VCClient.Client, "bogus") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("GetChildResourcePool", func() { + It("returns success", func() { + // Quick way for a child RP is to create a VMSetResourcePolicy. + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + + childRP, err := vcenter.GetChildResourcePool(ctx, nsRP, childRPName) + Expect(err).ToNot(HaveOccurred()) + Expect(childRP).ToNot(BeNil()) + + objRef, err := ctx.Finder.ObjectReference(ctx, childRP.Reference()) + Expect(err).ToNot(HaveOccurred()) + childRP, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + Expect(childRP.Name()).To(Equal(resourcePolicy.Name)) + }) + + It("returns error when child RP does not exist", func() { + childRP, err := vcenter.GetChildResourcePool(ctx, nsRP, "bogus") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found under parent ResourcePool")) + Expect(childRP).To(BeNil()) + }) + }) +} + +func createDeleteExistResourcePoolChild() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + nsRP *object.ResourcePool + + parentRPMoID string + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + nsRP = ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + + parentRPMoID = nsRP.Reference().Value + + resourcePolicy, _ = ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + nsRP = nil + parentRPMoID = "" + resourcePolicy = nil + }) + + Context("CreateOrUpdateChildResourcePool", func() { + It("creates child ResourcePool", func() { + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + Expect(childMoID).ToNot(BeEmpty()) + + By("returns success when child ResourcePool already exists", func() { + moID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + Expect(moID).To(Equal(childMoID)) + }) + + By("child ResourcePool is found by MoID", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, childMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(rp.Reference().Value).To(Equal(childMoID)) + }) + }) + + It("returns error when when parent ResourcePool MoID does not exist", func() { + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, "bogus", &resourcePolicy.Spec.ResourcePool) + Expect(err).To(HaveOccurred()) + Expect(childMoID).To(BeEmpty()) + }) + }) + + Context("DoesChildResourcePoolExist", func() { + It("returns true when child ResourcePool exists", func() { + childName := resourcePolicy.Spec.ResourcePool.Name + + _, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("returns false when child ResourcePool does not exist", func() { + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, parentRPMoID, "bogus") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + It("returns error when parent ResourcePool MoID does not exist", func() { + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, "bogus", "bogus") + Expect(err).To(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("DeleteChildResourcePool", func() { + It("deletes child ResourcePool", func() { + childName := resourcePolicy.Spec.ResourcePool.Name + + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + + err = vcenter.DeleteChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + + By("child ResourcePool is not found by MoID", func() { + _, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, childMoID) + Expect(err).To(HaveOccurred()) + }) + + By("NoOp when child does not exist", func() { + err := vcenter.DeleteChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go new file mode 100644 index 000000000..63d223192 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go @@ -0,0 +1,30 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() + +func vcSimTests() { + Describe("Cluster", clusterTests) + Describe("Folder", folderTests) + Describe("GetVM", getVMTests) + Describe("Host", hostTests) + Describe("ResourcePool", resourcePoolTests) +} + +func TestVCenter(t *testing.T) { + suite.Register(t, "VMProvider VCenter Tests", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go new file mode 100644 index 000000000..ff752d426 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go @@ -0,0 +1,34 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + + "github.com/vmware/govmomi/object" +) + +// GetVMClusterComputeResource returns the VM's ClusterComputeResource. +func GetVMClusterComputeResource( + ctx context.Context, + vcVM *object.VirtualMachine) (*object.ClusterComputeResource, error) { + + rp, err := vcVM.ResourcePool(ctx) + if err != nil { + return nil, err + } + + ccrRef, err := rp.Owner(ctx) + if err != nil { + return nil, err + } + + cluster, ok := ccrRef.(*object.ClusterComputeResource) + if !ok { + return nil, fmt.Errorf("VM Owner is not a ClusterComputeResource but %T", ccrRef) + } + + return cluster, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go new file mode 100644 index 000000000..55e4a5842 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func ccrTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Returns VM ClusterComputeResource", func() { + ccr, err := virtualmachine.GetVMClusterComputeResource(ctx, vcVM) + Expect(err).ToNot(HaveOccurred()) + Expect(ccr).ToNot(BeNil()) + Expect(ccr.Reference()).To(Equal(ctx.GetSingleClusterCompute().Reference())) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go new file mode 100644 index 000000000..5ae236f88 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go @@ -0,0 +1,190 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" +) + +// CreateConfigSpec returns an initial ConfigSpec that is created by overlaying the +// base ConfigSpec with VM Class spec and other arguments. +// TODO: We eventually need to de-dupe much of this with the ConfigSpec manipulation that's later done +// in the "update" pre-power on path. That operates on a ConfigInfo so we'd need to populate that from +// the config we build here. +func CreateConfigSpec( + vmCtx context.VirtualMachineContextA2, + vmClassConfigSpec *types.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + vmImageStatus *vmopv1.VirtualMachineImageStatus, + minFreq uint64) *types.VirtualMachineConfigSpec { + + configSpec := types.VirtualMachineConfigSpec{} + + // If there is a class ConfigSpec, then that is our initial ConfigSpec. + if vmClassConfigSpec != nil { + configSpec = *vmClassConfigSpec + } + + configSpec.Name = vmCtx.VM.Name + if configSpec.Annotation == "" { + // If the class ConfigSpec doesn't specify any annotations, set the default one. + configSpec.Annotation = constants.VCVMAnnotation + } + // CPU and Memory configurations specified in the VM Class standalone fields take + // precedence over values in the config spec + configSpec.NumCPUs = int32(vmClassSpec.Hardware.Cpus) + configSpec.MemoryMB = MemoryQuantityToMb(vmClassSpec.Hardware.Memory) + configSpec.ManagedBy = &types.ManagedByInfo{ + ExtensionKey: constants.ManagedByExtensionKey, + Type: constants.ManagedByExtensionType, + } + + if val, ok := vmCtx.VM.Annotations[constants.FirmwareOverrideAnnotation]; ok { + configSpec.Firmware = val + } else if configSpec.Firmware == "" && vmImageStatus != nil { + // Use firmware type from the image if ConfigSpec doesn't have it. + configSpec.Firmware = vmImageStatus.Firmware + } + + // TODO: Otherwise leave as-is? Our ChangeBlockTracking could be better as a *bool. + if vmCtx.VM.Spec.Advanced.ChangeBlockTracking { + configSpec.ChangeTrackingEnabled = pointer.Bool(true) + } + + // Populate the CPU reservation and limits in the ConfigSpec if VAPI fields specify any. + // VM Class VAPI does not support Limits, so they will never be non nil. + // TODO: Remove limits: issues/56 + if res := vmClassSpec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + // TODO: Always override? + configSpec.CpuAllocation = &types.ResourceAllocationInfo{ + Shares: &types.SharesInfo{ + Level: types.SharesLevelNormal, + }, + } + + if !res.Requests.Cpu.IsZero() { + rsv := CPUQuantityToMhz(vmClassSpec.Policies.Resources.Requests.Cpu, minFreq) + configSpec.CpuAllocation.Reservation = &rsv + } + if !res.Limits.Cpu.IsZero() { + lim := CPUQuantityToMhz(vmClassSpec.Policies.Resources.Limits.Cpu, minFreq) + configSpec.CpuAllocation.Limit = &lim + } + } + + // Populate the memory reservation and limits in the ConfigSpec if VAPI fields specify any. + // TODO: Remove limits: issues/56 + if res := vmClassSpec.Policies.Resources; !res.Requests.Memory.IsZero() || !res.Limits.Memory.IsZero() { + // TODO: Always override? + configSpec.MemoryAllocation = &types.ResourceAllocationInfo{ + Shares: &types.SharesInfo{ + Level: types.SharesLevelNormal, + }, + } + + if !res.Requests.Memory.IsZero() { + rsv := MemoryQuantityToMb(vmClassSpec.Policies.Resources.Requests.Memory) + configSpec.MemoryAllocation.Reservation = &rsv + } + if !res.Limits.Memory.IsZero() { + lim := MemoryQuantityToMb(vmClassSpec.Policies.Resources.Limits.Memory) + configSpec.MemoryAllocation.Limit = &lim + } + } + + return &configSpec +} + +// CreateConfigSpecForPlacement creates a ConfigSpec that is suitable for Placement. +// baseConfigSpec will likely be - or at least derived from - the ConfigSpec returned by CreateConfigSpec above. +func CreateConfigSpecForPlacement( + vmCtx context.VirtualMachineContextA2, + baseConfigSpec *types.VirtualMachineConfigSpec, + storageClassesToIDs map[string]string) *types.VirtualMachineConfigSpec { + + // TODO: If placement chokes on EthCards w/o a backing yet (NSX-T) remove those entries here. + deviceChangeCopy := make([]types.BaseVirtualDeviceConfigSpec, len(baseConfigSpec.DeviceChange)) + copy(deviceChangeCopy, baseConfigSpec.DeviceChange) + + configSpec := *baseConfigSpec + configSpec.DeviceChange = deviceChangeCopy + + // Add a dummy disk for placement: PlaceVmsXCluster expects there to always be at least one disk. + // Until we're in a position to have the OVF envelope here, add a dummy disk satisfy it. + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024 * 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + Profile: []types.BaseVirtualMachineProfileSpec{ + &types.VirtualMachineDefinedProfileSpec{ + ProfileId: storageClassesToIDs[vmCtx.VM.Spec.StorageClass], + }, + }, + }) + + if lib.IsInstanceStorageFSSEnabled() { + isVolumes := instancestorage.FilterVolumes(vmCtx.VM) + + for idx, dev := range CreateInstanceStorageDiskDevices(isVolumes) { + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate, + Device: dev, + Profile: []types.BaseVirtualMachineProfileSpec{ + &types.VirtualMachineDefinedProfileSpec{ + ProfileId: storageClassesToIDs[isVolumes[idx].PersistentVolumeClaim.InstanceVolumeClaim.StorageClass], + ProfileData: &types.VirtualMachineProfileRawData{ + ExtensionKey: "com.vmware.vim.sps", + }, + }, + }, + }) + } + } + + // TODO: Add more devices and fields + // - boot disks from OVA + // - storage profile/class + // - PVC volumes + // - Network devices (meh for now b/c of wcp constraints) + // - anything in ExtraConfig matter here? + // - any way to do the cluster modules for anti-affinity? + // - whatever else I'm forgetting + + return &configSpec +} + +// ConfigSpecFromVMClassDevices creates a ConfigSpec that adds the standalone hardware devices from +// the VMClass if any. This ConfigSpec will be used as the class ConfigSpec to CreateConfigSpec, with +// the rest of the class fields - like CPU count - applied on top. +func ConfigSpecFromVMClassDevices(vmClassSpec *vmopv1.VirtualMachineClassSpec) *types.VirtualMachineConfigSpec { + devsFromClass := CreatePCIDevicesFromVMClass(vmClassSpec.Hardware.Devices) + if len(devsFromClass) == 0 { + return nil + } + + configSpec := &types.VirtualMachineConfigSpec{} + for _, dev := range devsFromClass { + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: dev, + }) + } + return configSpec +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go new file mode 100644 index 000000000..0dd01e72c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go @@ -0,0 +1,278 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + vimtypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("CreateConfigSpec", func() { + const vmName = "dummy-vm" + + var ( + vmCtx context.VirtualMachineContextA2 + vmClassSpec *vmopv1.VirtualMachineClassSpec + vmImageStatus *vmopv1.VirtualMachineImageStatus + minCPUFreq uint64 + configSpec *vimtypes.VirtualMachineConfigSpec + classConfigSpec *vimtypes.VirtualMachineConfigSpec + err error + ) + + BeforeEach(func() { + vmClass := builder.DummyVirtualMachineClassA2() + vmClassSpec = &vmClass.Spec + vmImageStatus = &vmopv1.VirtualMachineImageStatus{Firmware: "efi"} + minCPUFreq = 2500 + + vm := builder.DummyVirtualMachineA2() + vm.Name = vmName + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithValues("vmName", vm.GetName()), + VM: vm, + } + }) + + It("Basic ConfigSpec assertions", func() { + configSpec = virtualmachine.CreateConfigSpec( + vmCtx, + nil, + vmClassSpec, + vmImageStatus, + minCPUFreq) + + Expect(configSpec).ToNot(BeNil()) + Expect(err).To(BeNil()) + Expect(configSpec.Name).To(Equal(vmName)) + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.NumCPUs).To(BeEquivalentTo(vmClassSpec.Hardware.Cpus)) + Expect(configSpec.MemoryMB).To(BeEquivalentTo(4 * 1024)) + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.Firmware).To(Equal(vmImageStatus.Firmware)) + }) + + Context("Use VM Class ConfigSpec", func() { + BeforeEach(func() { + classConfigSpec = &vimtypes.VirtualMachineConfigSpec{ + Name: "dont-use-this-dummy-VM", + Annotation: "test-annotation", + DeviceChange: []vimtypes.BaseVirtualDeviceConfigSpec{ + &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimtypes.VirtualE1000{ + VirtualEthernetCard: vimtypes.VirtualEthernetCard{ + VirtualDevice: vimtypes.VirtualDevice{ + Key: 4000, + }, + }, + }, + }, + }, + } + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.CreateConfigSpec( + vmCtx, + classConfigSpec, + vmClassSpec, + vmImageStatus, + minCPUFreq) + Expect(configSpec).ToNot(BeNil()) + }) + + It("Returns expected config spec", func() { + Expect(configSpec.Name).To(Equal(vmName)) + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.Annotation).To(Equal("test-annotation")) + Expect(configSpec.NumCPUs).To(BeEquivalentTo(vmClassSpec.Hardware.Cpus)) + Expect(configSpec.MemoryMB).To(BeEquivalentTo(4 * 1024)) + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.Firmware).To(Equal(vmImageStatus.Firmware)) + Expect(configSpec.DeviceChange).To(HaveLen(1)) + dSpec := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + _, ok := dSpec.Device.(*vimtypes.VirtualE1000) + Expect(ok).To(BeTrue()) + }) + }) +}) + +var _ = Describe("CreateConfigSpecForPlacement", func() { + + var ( + vmCtx context.VirtualMachineContextA2 + storageClassesToIDs map[string]string + baseConfigSpec *vimtypes.VirtualMachineConfigSpec + configSpec *vimtypes.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + baseConfigSpec = &vimtypes.VirtualMachineConfigSpec{} + storageClassesToIDs = map[string]string{} + + vm := builder.DummyVirtualMachineA2() + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithValues("vmName", vm.GetName()), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.CreateConfigSpecForPlacement( + vmCtx, + baseConfigSpec, + storageClassesToIDs) + Expect(configSpec).ToNot(BeNil()) + }) + + Context("Returns expected ConfigSpec", func() { + BeforeEach(func() { + baseConfigSpec = &vimtypes.VirtualMachineConfigSpec{ + Name: "dummy-VM", + Annotation: "test-annotation", + NumCPUs: 42, + MemoryMB: 4096, + Firmware: "secret-sauce", + DeviceChange: []vimtypes.BaseVirtualDeviceConfigSpec{ + &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimtypes.VirtualPCIPassthrough{ + VirtualDevice: vimtypes.VirtualDevice{ + Backing: &vimtypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + }, + } + }) + + It("Placement ConfigSpec contains expected field set sans ethernet device from class config spec", func() { + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.Annotation).To(Equal(baseConfigSpec.Annotation)) + Expect(configSpec.NumCPUs).To(Equal(baseConfigSpec.NumCPUs)) + Expect(configSpec.MemoryMB).To(Equal(baseConfigSpec.MemoryMB)) + Expect(configSpec.CpuAllocation).To(Equal(baseConfigSpec.CpuAllocation)) + Expect(configSpec.MemoryAllocation).To(Equal(baseConfigSpec.MemoryAllocation)) + Expect(configSpec.Firmware).To(Equal(baseConfigSpec.Firmware)) + + Expect(configSpec.DeviceChange).To(HaveLen(2)) + dSpec := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + _, ok := dSpec.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok).To(BeTrue()) + dSpec1 := configSpec.DeviceChange[1].GetVirtualDeviceConfigSpec() + _, ok = dSpec1.Device.(*vimtypes.VirtualDisk) + Expect(ok).To(BeTrue()) + }) + }) + + Context("When InstanceStorage is configured", func() { + const storagePolicyID = "storage-id-42" + var oldIsInstanceStorageFSSEnabled func() bool + + BeforeEach(func() { + oldIsInstanceStorageFSSEnabled = lib.IsInstanceStorageFSSEnabled + lib.IsInstanceStorageFSSEnabled = func() bool { return true } + + builder.AddDummyInstanceStorageVolumeA2(vmCtx.VM) + storageClassesToIDs[builder.DummyStorageClassName] = storagePolicyID + }) + + AfterEach(func() { + lib.IsInstanceStorageFSSEnabled = oldIsInstanceStorageFSSEnabled + }) + + It("ConfigSpec contains expected InstanceStorage devices", func() { + Expect(configSpec.DeviceChange).To(HaveLen(3)) + assertInstanceStorageDeviceChange(configSpec.DeviceChange[1], 256, storagePolicyID) + assertInstanceStorageDeviceChange(configSpec.DeviceChange[2], 512, storagePolicyID) + }) + }) +}) + +var _ = Describe("ConfigSpecFromVMClassDevices", func() { + + var ( + vmClassSpec *vmopv1.VirtualMachineClassSpec + configSpec *vimtypes.VirtualMachineConfigSpec + ) + + Context("when Class specifies GPU/DDPIO in Hardware", func() { + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + + vmClassSpec.Hardware.Devices.VGPUDevices = []vmopv1.VGPUDevice{{ + ProfileName: "createplacementspec-profile", + }} + + vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices = []vmopv1.DynamicDirectPathIODevice{{ + VendorID: 20, + DeviceID: 30, + CustomLabel: "createplacementspec-label", + }} + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.ConfigSpecFromVMClassDevices(vmClassSpec) + }) + + It("Returns expected ConfigSpec", func() { + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.DeviceChange).To(HaveLen(2)) // One each for GPU an DDPIO above + + dSpec1 := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + dev1, ok := dSpec1.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok).To(BeTrue()) + pciDev1 := dev1.GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*vimtypes.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).To(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal(vmClassSpec.Hardware.Devices.VGPUDevices[0].ProfileName)) + + dSpec2 := configSpec.DeviceChange[1].GetVirtualDeviceConfigSpec() + dev2, ok2 := dSpec2.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok2).To(BeTrue()) + pciDev2 := dev2.GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*vimtypes.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).To(BeTrue()) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(BeEquivalentTo(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].DeviceID)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(BeEquivalentTo(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].VendorID)) + Expect(pciBacking2.CustomLabel).To(Equal(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].CustomLabel)) + }) + }) +}) + +func assertInstanceStorageDeviceChange( + deviceChange vimtypes.BaseVirtualDeviceConfigSpec, + expectedSizeGB int, + expectedStoragePolicyID string) { + + dc := deviceChange.GetVirtualDeviceConfigSpec() + Expect(dc.Operation).To(Equal(vimtypes.VirtualDeviceConfigSpecOperationAdd)) + Expect(dc.FileOperation).To(Equal(vimtypes.VirtualDeviceConfigSpecFileOperationCreate)) + + dev, ok := dc.Device.(*vimtypes.VirtualDisk) + Expect(ok).To(BeTrue()) + Expect(dev.CapacityInBytes).To(BeEquivalentTo(expectedSizeGB * 1024 * 1024 * 1024)) + + Expect(dc.Profile).To(HaveLen(1)) + profile, ok := dc.Profile[0].(*vimtypes.VirtualMachineDefinedProfileSpec) + Expect(ok).To(BeTrue()) + Expect(profile.ProfileId).To(Equal(expectedStoragePolicyID)) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go new file mode 100644 index 000000000..06325c5e6 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go @@ -0,0 +1,18 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "math" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func MemoryQuantityToMb(q resource.Quantity) int64 { + return int64(math.Ceil(float64(q.Value()) / float64(1024*1024))) +} + +func CPUQuantityToMhz(q resource.Quantity, cpuFreqMhz uint64) int64 { + return int64(math.Ceil(float64(q.MilliValue()) * float64(cpuFreqMhz) / float64(1000))) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go new file mode 100644 index 000000000..cb53717eb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var _ = Describe("CPUQuantityToMhz", func() { + + Context("Convert CPU units from milli-cores to MHz", func() { + It("return whole number for non-integer CPU quantity", func() { + q, err := resource.ParseQuantity("500m") + Expect(err).NotTo(HaveOccurred()) + freq := virtualmachine.CPUQuantityToMhz(q, 3225) + expectVal := int64(1613) + Expect(freq).Should(BeNumerically("==", expectVal)) + }) + + It("return whole number for integer CPU quantity", func() { + q, err := resource.ParseQuantity("1000m") + Expect(err).NotTo(HaveOccurred()) + freq := virtualmachine.CPUQuantityToMhz(q, 3225) + expectVal := int64(3225) + Expect(freq).Should(BeNumerically("==", expectVal)) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go new file mode 100644 index 000000000..7641eaacb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" +) + +func DeleteVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine) error { + + if _, err := vmutil.SetAndWaitOnPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + vcVM.Client(), + vmutil.ManagedObjectFromObject(vcVM), + false, + types.VirtualMachinePowerStatePoweredOff, + vmutil.ParsePowerOpMode(string(vmCtx.VM.Spec.PowerOffMode))); err != nil { + + return err + } + + t, err := vcVM.Destroy(vmCtx) + if err != nil { + return err + } + + if taskInfo, err := t.WaitForResult(vmCtx); err != nil { + if taskInfo != nil { + vmCtx.Logger.V(5).Error(err, "destroy VM task failed", "taskInfo", taskInfo) + } + return errors.Wrapf(err, "destroy VM task failed") + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go new file mode 100644 index 000000000..3be4c39f5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func deleteTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), + VM: builder.DummyVirtualMachineA2(), + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Deletes VM that is off", func() { + moID := vcVM.Reference().Value + Expect(ctx.GetVMFromMoID(moID)).ToNot(BeNil()) + + t, err := vcVM.PowerOff(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Wait(ctx)).To(Succeed()) + + err = virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) + Expect(err).ToNot(HaveOccurred()) + + Expect(ctx.GetVMFromMoID(moID)).To(BeNil()) + }) + + It("Deletes VM that is on", func() { + moID := vcVM.Reference().Value + Expect(ctx.GetVMFromMoID(moID)).ToNot(BeNil()) + + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + err = virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) + Expect(err).ToNot(HaveOccurred()) + + Expect(ctx.GetVMFromMoID(moID)).To(BeNil()) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go new file mode 100644 index 000000000..1152ea435 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go @@ -0,0 +1,100 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + vimTypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +const ( + // A negative device range is traditionally used. + pciDevicesStartDeviceKey = int32(-200) + instanceStorageStartDeviceKey = int32(-300) +) + +func CreatePCIPassThroughDevice(deviceKey int32, backingInfo vimTypes.BaseVirtualDeviceBackingInfo) vimTypes.BaseVirtualDevice { + device := &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Key: deviceKey, + Backing: backingInfo, + }, + } + return device +} + +// CreatePCIDevicesFromConfigSpec creates vim25 VirtualDevices from the specified list of PCI devices from the VM Class ConfigSpec. +func CreatePCIDevicesFromConfigSpec(pciDevsFromConfigSpec []*vimTypes.VirtualPCIPassthrough) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(pciDevsFromConfigSpec)) + + deviceKey := pciDevicesStartDeviceKey + + for i := range pciDevsFromConfigSpec { + dev := pciDevsFromConfigSpec[i] + dev.Key = deviceKey + devices = append(devices, dev) + deviceKey-- + } + + return devices +} + +// CreatePCIDevicesFromVMClass creates vim25 VirtualDevices from the specified list of PCI devices from VM Class spec. +func CreatePCIDevicesFromVMClass(pciDevicesFromVMClass vmopv1.VirtualDevices) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(pciDevicesFromVMClass.VGPUDevices)+len(pciDevicesFromVMClass.DynamicDirectPathIODevices)) + + deviceKey := pciDevicesStartDeviceKey + + for _, vGPU := range pciDevicesFromVMClass.VGPUDevices { + backingInfo := &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: vGPU.ProfileName, + } + dev := CreatePCIPassThroughDevice(deviceKey, backingInfo) + devices = append(devices, dev) + deviceKey-- + } + + for _, dynamicDirectPath := range pciDevicesFromVMClass.DynamicDirectPathIODevices { + allowedDev := vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: int32(dynamicDirectPath.VendorID), + DeviceId: int32(dynamicDirectPath.DeviceID), + } + backingInfo := &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev}, + CustomLabel: dynamicDirectPath.CustomLabel, + } + dev := CreatePCIPassThroughDevice(deviceKey, backingInfo) + devices = append(devices, dev) + deviceKey-- + } + + return devices +} + +func CreateInstanceStorageDiskDevices(isVolumes []vmopv1.VirtualMachineVolume) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(isVolumes)) + deviceKey := instanceStorageStartDeviceKey + + for _, volume := range isVolumes { + device := &vimTypes.VirtualDisk{ + CapacityInBytes: volume.PersistentVolumeClaim.InstanceVolumeClaim.Size.Value(), + VirtualDevice: vimTypes.VirtualDevice{ + Key: deviceKey, + Backing: &vimTypes.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(false), + }, + }, + VDiskId: &vimTypes.ID{ + Id: constants.InstanceStorageVDiskID, + }, + } + devices = append(devices, device) + deviceKey-- + } + + return devices +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go new file mode 100644 index 000000000..424f8b527 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go @@ -0,0 +1,25 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +func GetGuestHeartBeatStatus( + ctx context.Context, + vm *object.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { + + var o mo.VirtualMachine + if err := vm.Properties(ctx, vm.Reference(), []string{"guestHeartbeatStatus"}, &o); err != nil { + return "", err + } + + return vmopv1.GuestHeartbeatStatus(o.GuestHeartbeatStatus), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go new file mode 100644 index 000000000..f1f13db2b --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go @@ -0,0 +1,64 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "fmt" + "net/http" + + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vapi/vcenter" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +const ( + sourceVirtualMachineType = "VirtualMachine" + + // vAPICtxActIDHttpHeader represents the http header in vAPI to pass down the activation ID. + vAPICtxActIDHttpHeader = "vapi-ctx-actid" + + itemDescriptionFormat = "virtualmachinepublishrequest.vmoperator.vmware.com: %s\n" +) + +func CreateOVF( + vmCtx context.VirtualMachineContextA2, + client *rest.Client, + vmPubReq *vmopv1.VirtualMachinePublishRequest, + cl *imgregv1a1.ContentLibrary, + actID string) (string, error) { + + // Use VM Operator specific description so that we can link published items + // to the vmPub if anything unexpected happened. + descriptionPrefix := fmt.Sprintf(itemDescriptionFormat, string(vmPubReq.UID)) + createSpec := vcenter.CreateSpec{ + Name: vmPubReq.Status.TargetRef.Item.Name, + Description: descriptionPrefix + vmPubReq.Status.TargetRef.Item.Description, + } + + source := vcenter.ResourceID{ + Type: sourceVirtualMachineType, + Value: vmCtx.VM.Status.UniqueID, + } + + target := vcenter.LibraryTarget{ + LibraryID: string(cl.Spec.UUID), + } + + ovf := vcenter.OVF{ + Spec: createSpec, + Source: source, + Target: target, + } + + vmCtx.Logger.Info("Creating OVF from VM", "spec", ovf, "actId", actID) + + // Use vmpublish uid as the act id passed down to the content library service, so that we can track + // the task status by the act id. + ctxHeader := client.WithHeader(vmCtx, http.Header{vAPICtxActIDHttpHeader: []string{actID}}) + return vcenter.NewManager(client).CreateOVF(ctxHeader, ovf) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go new file mode 100644 index 000000000..657f4de15 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func publishTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vm *vmopv1.VirtualMachine + cl *imgregv1a1.ContentLibrary + vmPub *vmopv1.VirtualMachinePublishRequest + vmCtx context.VirtualMachineContextA2 + vmPubCtx context.VirtualMachinePublishRequestContext + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true, WithContentLibrary: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + + vm = builder.DummyVirtualMachineA2() + vm.Status.UniqueID = vcVM.Reference().Value + cl = builder.DummyContentLibrary("dummy-cl", "dummy-ns", ctx.ContentLibraryID) + vmPub = builder.DummyVirtualMachinePublishRequestA2("dummy-vmpub", "dummy-ns", + vcVM.Name(), "dummy-item-name", "dummy-cl") + vmPub.Status.SourceRef = &vmPub.Spec.Source + vmPub.Status.TargetRef = &vmPub.Spec.Target + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), + VM: vm, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Publishes VM that is off", func() { + t, err := vcVM.PowerOff(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Wait(ctx)).To(Succeed()) + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) + + It("Publishes VM that is on", func() { + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) + + // TODO: update after vcsim bug is resolved. + // Currently if cl doesn't exist, vcsim set notFound http code + // but doesn't return immediately, which cause a panic error. + XIt("returns error if target content library does not exist", func() { + vmPubCtx.ContentLibrary.Spec.UUID = "12345" + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).To(HaveOccurred()) + Expect(itemID).To(BeEmpty()) + }) + + // TODO: vcsim currently doesn't check if an item already exists in the cl. + XIt("returns error if target content library item already exists", func() { + vmPubCtx.VMPublishRequest.Spec.Target.Item.Name = ctx.ContentLibraryImageName + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go new file mode 100644 index 000000000..8ba9c7376 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go @@ -0,0 +1,41 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/storage" +) + +// GetDefaultDiskProvisioningType gets the default disk provisioning type specified for the VM. +func GetDefaultDiskProvisioningType( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + storageProfileID string) (string, error) { + + switch vmCtx.VM.Spec.Advanced.DefaultVolumeProvisioningMode { + case vmopv1.VirtualMachineVolumeProvisioningModeThin: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil + case vmopv1.VirtualMachineVolumeProvisioningModeThick: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThick), nil + case vmopv1.VirtualMachineVolumeProvisioningModeThickEagerZero: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeEagerZeroedThick), nil + } + + if storageProfileID != "" { + provisioning, err := storage.GetDiskProvisioningForProfile(vmCtx, vcClient, storageProfileID) + if err != nil { + return "", err + } + if provisioning != "" { + return provisioning, nil + } + } + + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go new file mode 100644 index 000000000..12c5c35fd --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ClusterComputeResource", ccrTests) + Describe("Delete", deleteTests) + Describe("Publish", publishTests) +} + +var suite = builder.NewTestSuite() + +func TestClusterModules(t *testing.T) { + suite.Register(t, "vSphere Provider VirtualMachine Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go new file mode 100644 index 000000000..cc311b468 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go @@ -0,0 +1,65 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha512" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +func GetWebConsoleTicket( + vmCtx context.VirtualMachineContextA2, + vm *object.VirtualMachine, + pubKey string) (string, error) { + + vmCtx.Logger.V(5).Info("GetWebMKSTicket") + + ticket, err := vm.AcquireTicket(vmCtx, string(types.VirtualMachineTicketTypeWebmks)) + if err != nil { + return "", err + } + + url := fmt.Sprintf("wss://%s:%d/ticket/%s", ticket.Host, ticket.Port, ticket.Ticket) + return EncryptWebMKS(pubKey, url) +} + +func EncryptWebMKS(pubKey string, plaintext string) (string, error) { + block, _ := pem.Decode([]byte(pubKey)) + if block == nil || block.Type != "PUBLIC KEY" { + return "", errors.New("failed to decode PEM block containing public key") + } + pub, err := x509.ParsePKCS1PublicKey(block.Bytes) + if err != nil { + return "", err + } + cipherbytes, err := rsa.EncryptOAEP(sha512.New(), rand.Reader, pub, []byte(plaintext), nil) + if err != nil { + return "", err + } + ciphertext := base64.StdEncoding.EncodeToString(cipherbytes) + return ciphertext, nil +} + +func DecryptWebMKS(privKey *rsa.PrivateKey, ciphertext string) (string, error) { + decoded, err := base64.StdEncoding.DecodeString(ciphertext) + if err != nil { + return "", err + } + decrypted, err := rsa.DecryptOAEP(sha512.New(), rand.Reader, privKey, decoded, nil) + if err != nil { + return "", err + } + return string(decrypted), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go new file mode 100644 index 000000000..50a05cc72 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "crypto/rsa" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("Webconsole Ticket", func() { + + Context("EncryptWebMKS", func() { + var ( + privateKey *rsa.PrivateKey + publicKeyPem string + ) + + BeforeEach(func() { + privateKey, publicKeyPem = builder.WebConsoleRequestKeyPair() + }) + + It("Encrypts a string correctly", func() { + plaintext := "HelloWorld2" + ciphertext, err := virtualmachine.EncryptWebMKS(publicKeyPem, plaintext) + Expect(err).ShouldNot(HaveOccurred()) + decrypted, err := virtualmachine.DecryptWebMKS(privateKey, ciphertext) + Expect(err).ShouldNot(HaveOccurred()) + Expect(decrypted).To(Equal(plaintext)) + }) + + It("Error on invalid public key", func() { + plaintext := "HelloWorld3" + _, err := virtualmachine.EncryptWebMKS("invalid-pub-key", plaintext) + Expect(err).Should(HaveOccurred()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go new file mode 100644 index 000000000..1546399e8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go @@ -0,0 +1,265 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/task" + vimTypes "github.com/vmware/govmomi/vim25/types" + apiEquality "k8s.io/apimachinery/pkg/api/equality" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" +) + +const ( + // OvfEnvironmentTransportGuestInfo is the OVF transport type that uses + // GuestInfo. The other valid type is "iso". + OvfEnvironmentTransportGuestInfo = "com.vmware.guestInfo" +) + +type BootstrapData struct { + Data map[string]string + VAppData map[string]string + VAppExData map[string]map[string]string +} + +type TemplateRenderFunc func(string, string) string + +type BootstrapArgs struct { + BootstrapData + + TemplateRenderFn TemplateRenderFunc + NetworkResults network.NetworkInterfaceResults + Hostname string + DNSServers []string + SearchSuffixes []string +} + +func DoBootstrap( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + k8sClient ctrl.Client, + networkResults network.NetworkInterfaceResults, + bootstrapData BootstrapData) error { + + bootstrap := &vmCtx.VM.Spec.Bootstrap + cloudInit := bootstrap.CloudInit + linuxPrep := bootstrap.LinuxPrep + sysPrep := bootstrap.Sysprep + vAppConfig := bootstrap.VAppConfig + + bootstrapArgs, err := getBootstrapArgs(vmCtx, k8sClient, cloudInit != nil, networkResults, bootstrapData) + if err != nil { + return err + } + + if vAppConfig != nil { + // I think the intention was to only apply this to vAppData. Old code would apply it to entire + // Data map but for like SysPrep that data may be base64/gzip'd, and we'd do the template stuff + // prior to plain texting it. + bootstrapArgs.TemplateRenderFn = GetTemplateRenderFunc(vmCtx, bootstrapArgs) + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + var customSpec *vimTypes.CustomizationSpec + + switch { + case cloudInit != nil: + configSpec, customSpec, err = BootStrapCloudInit(vmCtx, config, cloudInit, bootstrapArgs) + case linuxPrep != nil: + configSpec, customSpec, err = BootStrapLinuxPrep(vmCtx, config, linuxPrep, vAppConfig, bootstrapArgs) + case sysPrep != nil: + configSpec, customSpec, err = BootstrapSysPrep(vmCtx, config, sysPrep, vAppConfig, bootstrapArgs) + case vAppConfig != nil: + configSpec, customSpec, err = BootstrapVAppConfig(vmCtx, config, vAppConfig, bootstrapArgs) + default: + // Old code fell back to LinuxPrep. Is that really appropriate anymore? + linuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{HardwareClockIsUTC: true} + configSpec, customSpec, err = BootStrapLinuxPrep(vmCtx, config, linuxPrep, nil, bootstrapArgs) + } + + if err != nil { + return fmt.Errorf("failed to create bootstrap data: %w", err) + } + + if configSpec != nil { + err := doReconfigure(vmCtx, vcVM, configSpec) + if err != nil { + return fmt.Errorf("boostrap reconfigure failed: %w", err) + } + } + + if customSpec != nil { + err := doCustomize(vmCtx, vcVM, config, customSpec) + if err != nil { + return fmt.Errorf("boostrap customize failed: %w", err) + } + } + + return nil +} + +func getBootstrapArgs( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrl.Client, + isCloudInit bool, + networkResults network.NetworkInterfaceResults, + bootstrapData BootstrapData) (*BootstrapArgs, error) { + + bootstrapArgs := BootstrapArgs{ + BootstrapData: bootstrapData, + NetworkResults: networkResults, + Hostname: vmCtx.VM.Spec.Network.HostName, + } + + if bootstrapArgs.Hostname == "" { + bootstrapArgs.Hostname = vmCtx.VM.Name + } + + // If the VM is missing DNS info - that is, it did not specify DNS for the interfaces - populate that + // now from the SV global configuration. Note that the VM is probably OK as long as at least one + // interface has DNS info, but we would previously set it for every interface so keep doing that + // here. Similarly, we didn't populate SearchDomains for non-TKG VMs so we don't here either. This is + // all a little nuts & complicated and probably not correct for every situation. + isTKG := hasTKGLabels(vmCtx.VM.Labels) + missingDNSInfo := false + for _, r := range networkResults.Results { + if r.DHCP4 || r.DHCP6 { + continue + } + + if len(r.Nameservers) == 0 || (isTKG && len(r.SearchDomains) == 0) { + missingDNSInfo = true + break + } + } + + if missingDNSInfo { + nameservers, searchSuffixes, err := config.GetDNSInformationFromConfigMap(k8sClient) + if err != nil && ctrl.IgnoreNotFound(err) != nil { + // This ConfigMap doesn't exist in certain test envs. + return nil, err + } + + // GOSC will use these for its global config. + bootstrapArgs.DNSServers = nameservers + bootstrapArgs.SearchSuffixes = searchSuffixes + + if isCloudInit { + // Previously we would apply the global DNS config to every interface so do that here too. + for i := range networkResults.Results { + r := &networkResults.Results[i] + + if r.DHCP4 || r.DHCP6 { + continue + } + + if len(r.Nameservers) == 0 { + r.Nameservers = nameservers + } + if isTKG && len(r.SearchDomains) == 0 { + r.SearchDomains = searchSuffixes + } + } + } + } + + return &bootstrapArgs, nil +} + +func hasTKGLabels(vmLabels map[string]string) bool { + const ( + // CAPWClusterRoleLabelKey is the key for the label applied to a VM that was + // created by CAPW. + CAPWClusterRoleLabelKey = "capw.vmware.com/cluster.role" //nolint:gosec + + // CAPVClusterRoleLabelKey is the key for the label applied to a VM that was + // created by CAPV. + CAPVClusterRoleLabelKey = "capv.vmware.com/cluster.role" + ) + + _, ok := vmLabels[CAPWClusterRoleLabelKey] + if !ok { + _, ok = vmLabels[CAPVClusterRoleLabelKey] + } + return ok +} + +func doReconfigure( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + configSpec *vimTypes.VirtualMachineConfigSpec) error { + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("Customization Reconfigure", "configSpec", configSpec) + + if err := resources.NewVMFromObject(vcVM).Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "customization reconfigure failed") + return err + } + } + + return nil +} + +func doCustomize( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + customSpec *vimTypes.CustomizationSpec) error { + + if vmCtx.VM.Annotations[constants.VSphereCustomizationBypassKey] == constants.VSphereCustomizationBypassDisable { + vmCtx.Logger.Info("Skipping vsphere customization because of vsphere-customization bypass annotation") + return nil + } + + if IsCustomizationPendingExtraConfig(config.ExtraConfig) { + vmCtx.Logger.Info("Skipping customization because it is already pending") + // TODO: We should really determine if the pending customization is stale, clear it + // if so, and then re-customize. Otherwise, the Customize call could perpetually fail + // preventing power on. + return nil + } + + vmCtx.Logger.Info("Customizing VM", "customizationSpec", *customSpec) + if err := resources.NewVMFromObject(vcVM).Customize(vmCtx, *customSpec); err != nil { + // isCustomizationPendingExtraConfig() above is supposed to prevent this error, but + // handle it explicitly here just in case so VM reconciliation can proceed. + if !isCustomizationPendingError(err) { + return err + } + } + + return nil +} + +func IsCustomizationPendingExtraConfig(extraConfig []vimTypes.BaseOptionValue) bool { + for _, opt := range extraConfig { + if optValue := opt.GetOptionValue(); optValue != nil { + if optValue.Key == constants.GOSCPendingExtraConfigKey { + return optValue.Value.(string) != "" + } + } + } + return false +} + +func isCustomizationPendingError(err error) bool { + if te, ok := err.(task.Error); ok { + if _, ok := te.Fault().(*vimTypes.CustomizationPending); ok { + return true + } + } + return false +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go new file mode 100644 index 000000000..9e46536d4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go @@ -0,0 +1,186 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + "strings" + + "github.com/vmware/govmomi/vim25/types" + "gopkg.in/yaml.v2" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +type CloudInitMetadata struct { + InstanceID string `yaml:"instance-id,omitempty"` + LocalHostname string `yaml:"local-hostname,omitempty"` + Hostname string `yaml:"hostname,omitempty"` + Network network.Netplan `yaml:"network,omitempty"` + PublicKeys string `yaml:"public-keys,omitempty"` +} + +func BootStrapCloudInit( + vmCtx context.VirtualMachineContextA2, + config *types.VirtualMachineConfigInfo, + cloudInitSpec *vmopv1.VirtualMachineBootstrapCloudInitSpec, + bsArgs *BootstrapArgs) (*types.VirtualMachineConfigSpec, *types.CustomizationSpec, error) { + + netPlan, err := network.NetPlanCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create NetPlan customization: %w", err) + } + + sshPublicKeys := bsArgs.BootstrapData.Data["ssh-public-keys"] + if len(cloudInitSpec.SSHAuthorizedKeys) > 0 { + sshPublicKeys = strings.Join(cloudInitSpec.SSHAuthorizedKeys, "\n") + } + + metadata, err := GetCloudInitMetadata(string(vmCtx.VM.UID), bsArgs.Hostname, netPlan, sshPublicKeys) + if err != nil { + return nil, nil, err + } + + var userdata string + if cloudInitSpec.RawCloudConfig.Name != "" { + // Check for the 'user-data' key as per official contract and API documentation. + // Additionally, to support the cluster bootstrap data supplied by CAPBK's secret, + // we check for a 'value' key when 'user-data' is not supplied. + // The 'value' key lookup will eventually be deprecated. + for _, k := range []string{cloudInitSpec.RawCloudConfig.Key, "user-data", "value"} { + if k != "" { + userdata = bsArgs.BootstrapData.Data[k] + if userdata != "" { + break + } + } + } + + // NOTE: The old code didn't error out if userdata wasn't found, so keep going. + + } else { + return nil, nil, fmt.Errorf("TODO: inlined CloudConfig") + } + + var configSpec *types.VirtualMachineConfigSpec + var customSpec *types.CustomizationSpec + + switch vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] { + case constants.CloudInitTypeValueCloudInitPrep: + configSpec, customSpec, err = GetCloudInitPrepCustSpec(metadata, userdata) + case constants.CloudInitTypeValueGuestInfo, "": + fallthrough + default: + configSpec, err = GetCloudInitGuestInfoCustSpec(config, metadata, userdata) + } + + if err != nil { + return nil, nil, err + } + + return configSpec, customSpec, nil +} + +func GetCloudInitMetadata( + uid string, + hostname string, + netplan *network.Netplan, + sshPublicKeys string) (string, error) { + + metadata := &CloudInitMetadata{ + InstanceID: uid, + LocalHostname: hostname, + Hostname: hostname, + Network: *netplan, + PublicKeys: sshPublicKeys, + } + + metadataBytes, err := yaml.Marshal(metadata) + if err != nil { + return "", fmt.Errorf("yaml marshalling of cloud-init metadata failed: %w", err) + } + + return string(metadataBytes), nil +} + +func GetCloudInitPrepCustSpec( + metadata, userdata string) (*types.VirtualMachineConfigSpec, *types.CustomizationSpec, error) { + + if userdata != "" { + // Ensure the data is normalized first to plain-text. + plainText, err := util.TryToDecodeBase64Gzip([]byte(userdata)) + if err != nil { + return nil, nil, fmt.Errorf("decoding cloud-init prep userdata failed: %w", err) + } + + userdata = plainText + } + + // FIXME: This is the old behavior but isn't quite correct: we need current config, and only + // do this if the transport isn't already set. Otherwise, this always results in a Reconfigure. + configSpec := &types.VirtualMachineConfigSpec{ + VAppConfig: &types.VmConfigSpec{ + // Ensure the transport is guestInfo in case the VM does not have + // a CD-ROM device required to use the ISO transport. + OvfEnvironmentTransport: []string{OvfEnvironmentTransportGuestInfo}, + }, + } + + customSpec := &types.CustomizationSpec{ + Identity: &internal.CustomizationCloudinitPrep{ + Metadata: metadata, + Userdata: userdata, + }, + } + + return configSpec, customSpec, nil +} + +func GetCloudInitGuestInfoCustSpec( + config *types.VirtualMachineConfigInfo, + metadata, userdata string) (*types.VirtualMachineConfigSpec, error) { + + encodedMetadata, err := util.EncodeGzipBase64(metadata) + if err != nil { + return nil, fmt.Errorf("encoding cloud-init metadata failed: %w", err) + } + + extraConfig := map[string]string{ + constants.CloudInitGuestInfoMetadata: encodedMetadata, + constants.CloudInitGuestInfoMetadataEncoding: "gzip+base64", + } + + if userdata != "" { + // Ensure the data is normalized first to plain-text. + plainText, err := util.TryToDecodeBase64Gzip([]byte(userdata)) + if err != nil { + return nil, fmt.Errorf("decoding cloud-init userdata failed: %w", err) + } + + encodedUserdata, err := util.EncodeGzipBase64(plainText) + if err != nil { + return nil, fmt.Errorf("encoding cloud-init userdata failed: %w", err) + } + + extraConfig[constants.CloudInitGuestInfoUserdata] = encodedUserdata + extraConfig[constants.CloudInitGuestInfoUserdataEncoding] = "gzip+base64" + } + + // FIXME: This is the old behavior but isn't quite correct: we really need the current ExtraConfig, + // and then only add/update it if new or updated (so we don't customize with stale data). Also, + // always setting VAppConfigRemoved isn't correct: should only set it if the VM has vApp data. + // As-is we will always Reconfigure the VM. + configSpec := &types.VirtualMachineConfigSpec{} + configSpec.ExtraConfig = util.AppendNewExtraConfigValues(config.ExtraConfig, extraConfig) + // Remove the VAppConfig to ensure Cloud-Init inside the guest does not + // activate and prefer the OVF datasource over the VMware datasource. + configSpec.VAppConfigRemoved = types.NewBool(true) // FIXME + + return configSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go new file mode 100644 index 000000000..7badf04c8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go @@ -0,0 +1,398 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + "encoding/base64" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + "gopkg.in/yaml.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("CloudInit Bootstrap", func() { + const ( + cloudInitMetadata = "cloud-init-metadata" + cloudInitUserdata = "cloud-init-userdata" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + + metaData string + userData string + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + + // Set defaults. + metaData = cloudInitMetadata + userData = cloudInitUserdata + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + extraConfigToMap := func(input []types.BaseOptionValue) map[string]string { + output := make(map[string]string) + for _, opt := range input { + if optValue := opt.GetOptionValue(); optValue != nil { + // Only set string type values + if val, ok := optValue.Value.(string); ok { + output[optValue.Key] = val + } + } + } + return output + } + + // v1a1 tests really only tested the lower level functions individually. Those tests are ported after + // this Context, but we should focus more on testing via this just method. + Context("BootStrapCloudInit", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + cloudInitSpec *vmopv1.VirtualMachineBootstrapCloudInitSpec + ) + + BeforeEach(func() { + cloudInitSpec = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-init-bootstrap-test", + Namespace: "test-ns", + UID: "my-vm-uuid", + Annotations: map[string]string{}, + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootStrapCloudInit( + vmCtx, + configInfo, + cloudInitSpec, + &bsArgs, + ) + }) + + Context("CloudInit Inlined Config", func() { + It("Returns TODO", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("TODO")) + }) + }) + + Context("CloudInit Data Config", func() { + BeforeEach(func() { + cloudInitSpec.RawCloudConfig.Name = "my-data" + cloudInitSpec.RawCloudConfig.Key = "my-key" + bsArgs.Data[cloudInitSpec.RawCloudConfig.Key] = cloudInitUserdata + }) + + Context("Via CloudInitPrep", func() { + BeforeEach(func() { + vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] = constants.CloudInitTypeValueCloudInitPrep + }) + + It("Returns success", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec()).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport).To(HaveLen(1)) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).ToNot(BeEmpty()) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("Via GuestInfo", func() { + BeforeEach(func() { + vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] = constants.CloudInitTypeValueGuestInfo + }) + + It("Returns Success", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfigRemoved).ToNot(BeNil()) + Expect(*configSpec.VAppConfigRemoved).To(BeTrue()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + + Expect(custSpec).To(BeNil()) + }) + + Context("Via CAPBK userdata in 'value' key", func() { + const otherUserData = cloudInitUserdata + "CAPBK" + + BeforeEach(func() { + bsArgs.Data[cloudInitSpec.RawCloudConfig.Key] = "" + cloudInitSpec.RawCloudConfig.Key = "" + bsArgs.Data["value"] = otherUserData + }) + + It("Returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoUserdata)) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + + data, err := util.TryToDecodeBase64Gzip([]byte(extraConfig[constants.CloudInitGuestInfoUserdata])) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(Equal(otherUserData)) + }) + }) + }) + }) + }) + + Context("GetCloudInitMetadata", func() { + var ( + uid string + hostName string + netPlan *network.Netplan + sshPublicKeys string + + mdYaml string + err error + ) + + BeforeEach(func() { + uid = "my-uid" + hostName = "my-hostname" + netPlan = &network.Netplan{ + Version: 42, + Ethernets: map[string]network.NetplanEthernet{ + "eth0": { + SetName: "eth0", + }, + }, + } + sshPublicKeys = "my-ssh-key" + }) + + JustBeforeEach(func() { + mdYaml, err = vmlifecycle.GetCloudInitMetadata(uid, hostName, netPlan, sshPublicKeys) + }) + + It("DoIt", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(mdYaml).ToNot(BeEmpty()) + + ciMetadata := &vmlifecycle.CloudInitMetadata{} + Expect(yaml.Unmarshal([]byte(mdYaml), ciMetadata)).To(Succeed()) + + Expect(ciMetadata.InstanceID).To(Equal(uid)) + Expect(ciMetadata.Hostname).To(Equal(hostName)) + Expect(ciMetadata.PublicKeys).To(Equal(sshPublicKeys)) + Expect(ciMetadata.Network.Version).To(Equal(42)) + Expect(ciMetadata.Network.Ethernets).To(HaveKey("eth0")) + }) + }) + + Context("GetCloudInitGuestInfoCustSpec", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + err error + ) + + JustBeforeEach(func() { + configSpec, err = vmlifecycle.GetCloudInitGuestInfoCustSpec(configInfo, metaData, userData) + }) + + Context("VAppConfig Disabled", func() { + It("Should disable the VAppConfig", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfigRemoved).ToNot(BeNil()) + Expect(*configSpec.VAppConfigRemoved).To(BeTrue()) + }) + }) + + Context("No userdata", func() { + BeforeEach(func() { + userData = "" + }) + + It("ConfigSpec.ExtraConfig to only have metadata", func() { + Expect(configSpec).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(2)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With userdata", func() { + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + userData = base64.StdEncoding.EncodeToString([]byte(cloudInitUserdata)) + }) + + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With gzipped, base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + data, err := util.EncodeGzipBase64(cloudInitUserdata) + Expect(err).ToNot(HaveOccurred()) + userData = data + }) + + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + }) + + Context("GetCloudInitPrepCustSpec", func() { + var ( + custSpec *types.CustomizationSpec + ) + + JustBeforeEach(func() { + var configSpec *types.VirtualMachineConfigSpec + var err error + + configSpec, custSpec, err = vmlifecycle.GetCloudInitPrepCustSpec(metaData, userData) + Expect(err).ToNot(HaveOccurred()) + + // Validate that Cloud-Init Prep always uses the GuestInfo transport for the CI Prep's meta and user data. + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec()).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport).To(HaveLen(1)) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + }) + + Context("No userdata", func() { + BeforeEach(func() { + userData = "" + }) + + It("Cust spec to only have metadata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(BeEmpty()) + }) + }) + + Context("With userdata", func() { + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("With base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + userData = base64.StdEncoding.EncodeToString([]byte(cloudInitUserdata)) + }) + + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("With gzipped, base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + data, err := util.EncodeGzipBase64(cloudInitUserdata) + Expect(err).ToNot(HaveOccurred()) + userData = data + }) + + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go new file mode 100644 index 000000000..0f954fd0f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go @@ -0,0 +1,55 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +func BootStrapLinuxPrep( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + linuxPrepSpec *vmopv1.VirtualMachineBootstrapLinuxPrepSpec, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + nicSettingMap, err := network.GuestOSCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create GOSC NIC mappings: %w", err) + } + + customSpec := &vimTypes.CustomizationSpec{ + Identity: &vimTypes.CustomizationLinuxPrep{ + HostName: &vimTypes.CustomizationFixedName{ + Name: bsArgs.Hostname, + }, + TimeZone: linuxPrepSpec.TimeZone, + HwClockUTC: vimTypes.NewBool(linuxPrepSpec.HardwareClockIsUTC), + }, + GlobalIPSettings: vimTypes.CustomizationGlobalIPSettings{ + DnsSuffixList: bsArgs.SearchSuffixes, + DnsServerList: bsArgs.DNSServers, + }, + NicSettingMap: nicSettingMap, + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + if vAppConfigSpec != nil { + configSpec = &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + } + + return configSpec, customSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go new file mode 100644 index 000000000..06fcd4a12 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go @@ -0,0 +1,156 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("LinuxPrep Bootstrap", func() { + const ( + macAddr = "43-AB-B4-1B-7E-87" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("BootStrapLinuxPrep", func() { + + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + linuxPrepSpec *vmopv1.VirtualMachineBootstrapLinuxPrepSpec + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + ) + + BeforeEach(func() { + linuxPrepSpec = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + vAppConfigSpec = nil + + bsArgs.Hostname = "my-hostname" + bsArgs.SearchSuffixes = []string{"suffix1", "suffix2"} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: "192.168.1.1", + IPCIDR: "192.168.1.10/24", + IsIPv4: true, + }, + }, + }, + } + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "linux-prep-bootstrap-test", + Namespace: "test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootStrapLinuxPrep( + vmCtx, + configInfo, + linuxPrepSpec, + vAppConfigSpec, + &bsArgs, + ) + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).To(BeNil()) + + Expect(custSpec).ToNot(BeNil()) + Expect(custSpec.GlobalIPSettings.DnsServerList).To(Equal(bsArgs.DNSServers)) + Expect(custSpec.GlobalIPSettings.DnsSuffixList).To(Equal(bsArgs.SearchSuffixes)) + + linuxSpec := custSpec.Identity.(*types.CustomizationLinuxPrep) + hostName := linuxSpec.HostName.(*types.CustomizationFixedName).Name + Expect(hostName).To(Equal(bsArgs.Hostname)) + Expect(linuxSpec.TimeZone).To(Equal(linuxPrepSpec.TimeZone)) + Expect(linuxSpec.HwClockUTC).ToNot(BeNil()) + Expect(*linuxSpec.HwClockUTC).To(Equal(linuxPrepSpec.HardwareClockIsUTC)) + + Expect(custSpec.NicSettingMap).To(HaveLen(len(bsArgs.NetworkResults.Results))) + Expect(custSpec.NicSettingMap[0].MacAddress).To(Equal(macAddr)) + }) + + Context("when has vAppConfig", func() { + const key, value = "fooKey", "fooValue" + + BeforeEach(func() { + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(custSpec).ToNot(BeNil()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + vmCs := configSpec.VAppConfig.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go new file mode 100644 index 000000000..a8fe2baea --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go @@ -0,0 +1,79 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + + vimTypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/apimachinery/pkg/api/equality" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/sysprep" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +func BootstrapSysPrep( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + sysPrepSpec *vmopv1.VirtualMachineBootstrapSysprepSpec, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + var data string + + if equality.Semantic.DeepEqual(sysPrepSpec.Sysprep, sysprep.Sysprep{}) { + var err error + + key := "unattend" + if sysPrepSpec.RawSysprep.Key != "" { + key = sysPrepSpec.RawSysprep.Key + } + + data = bsArgs.BootstrapData.Data[key] + if data == "" { + return nil, nil, fmt.Errorf("no Sysprep XML data with key %q", key) + } + + // Ensure the data is normalized first to plain-text. + data, err = util.TryToDecodeBase64Gzip([]byte(data)) + if err != nil { + return nil, nil, fmt.Errorf("decoding Sysprep unattend XML failed: %w", err) + } + + } else { + return nil, nil, fmt.Errorf("TODO: inlined Sysprep") + } + + nicSettingMap, err := network.GuestOSCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create GSOC adapter mappings: %w", err) + } + + customSpec := &vimTypes.CustomizationSpec{ + Identity: &vimTypes.CustomizationSysprepText{ + Value: data, + }, + GlobalIPSettings: vimTypes.CustomizationGlobalIPSettings{ + DnsSuffixList: bsArgs.SearchSuffixes, + DnsServerList: bsArgs.DNSServers, + }, + NicSettingMap: nicSettingMap, + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + if vAppConfigSpec != nil { + configSpec = &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + } + + return configSpec, customSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go new file mode 100644 index 000000000..2a61f0ac7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("SysPrep Bootstrap", func() { + const ( + macAddr = "43-AB-B4-1B-7E-87" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("BootStrapSysPrep", func() { + const unattendXML = "dummy-unattend-xml" + + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + sysPrepSpec *vmopv1.VirtualMachineBootstrapSysprepSpec + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + ) + + BeforeEach(func() { + sysPrepSpec = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + vAppConfigSpec = nil + + bsArgs.Data["unattend"] = unattendXML + bsArgs.SearchSuffixes = []string{"suffix1", "suffix2"} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: "192.168.1.1", + IPCIDR: "192.168.1.10/24", + IsIPv4: true, + }, + }, + }, + } + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sys-prep-bootstrap-test", + Namespace: "test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootstrapSysPrep( + vmCtx, + configInfo, + sysPrepSpec, + vAppConfigSpec, + &bsArgs, + ) + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).To(BeNil()) + + Expect(custSpec).ToNot(BeNil()) + Expect(custSpec.GlobalIPSettings.DnsServerList).To(Equal(bsArgs.DNSServers)) + Expect(custSpec.GlobalIPSettings.DnsSuffixList).To(Equal(bsArgs.SearchSuffixes)) + + sysPrepText := custSpec.Identity.(*types.CustomizationSysprepText) + Expect(sysPrepText.Value).To(Equal(unattendXML)) + + Expect(custSpec.NicSettingMap).To(HaveLen(len(bsArgs.NetworkResults.Results))) + Expect(custSpec.NicSettingMap[0].MacAddress).To(Equal(macAddr)) + }) + + Context("when has vAppConfig", func() { + const key, value = "fooKey", "fooValue" + + BeforeEach(func() { + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(custSpec).ToNot(BeNil()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + vmCs := configSpec.VAppConfig.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go new file mode 100644 index 000000000..e80ee1042 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go @@ -0,0 +1,453 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "bytes" + "errors" + "fmt" + "net" + "strings" + "text/template" + + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +func GetTemplateRenderFunc( + vmCtx context.VirtualMachineContextA2, + bsArgs *BootstrapArgs, +) TemplateRenderFunc { + + // There is a lot of duplication here, especially since the "template" types are the same in v1a1 + // and v1a2. We've conflated a lot of things here making this all a little nuts. + + networkDevicesStatusV1A1 := toTemplateNetworkStatusV1A1(bsArgs) + networkStatusV1A1 := v1alpha1.NetworkStatus{ + Devices: networkDevicesStatusV1A1, + Nameservers: bsArgs.DNSServers, + } + + networkDevicesStatusV1A2 := toTemplateNetworkStatus(bsArgs) + networkStatusV1A2 := v1alpha2.NetworkStatus{ + Devices: networkDevicesStatusV1A2, + Nameservers: bsArgs.DNSServers, + } + + // Oh dear. The VM itself really should not have been included here. + v1a1VM := &v1alpha1.VirtualMachine{} + _ = v1a1VM.ConvertFrom(vmCtx.VM) + + templateData := struct { + V1alpha1 v1alpha1.VirtualMachineTemplate + V1alpha2 v1alpha2.VirtualMachineTemplate + }{ + V1alpha1: v1alpha1.VirtualMachineTemplate{ + Net: networkStatusV1A1, + VM: v1a1VM, + }, + V1alpha2: v1alpha2.VirtualMachineTemplate{ + Net: networkStatusV1A2, + VM: vmCtx.VM, + }, + } + + v1a1FuncMap := v1a1TemplateFunctions(networkStatusV1A1, networkDevicesStatusV1A1) + v1a2FuncMap := v1a2TemplateFunctions(networkStatusV1A2, networkDevicesStatusV1A2) + + // Include both but should probably leave out v1a2 if we can identify this was originally a v1a1 VM. + funcMap := template.FuncMap{} + for k, v := range v1a1FuncMap { + funcMap[k] = v + } + for k, v := range v1a2FuncMap { + funcMap[k] = v + } + + // Skip parsing when encountering escape character('\{',"\}") + normalizeStr := func(str string) string { + if strings.Contains(str, "\\{") || strings.Contains(str, "\\}") { + str = strings.ReplaceAll(str, "\\{", "{") + str = strings.ReplaceAll(str, "\\}", "}") + } + return str + } + + // TODO: Don't log, return errors instead. + renderTemplate := func(name, templateStr string) string { + templ, err := template.New(name).Funcs(funcMap).Parse(templateStr) + if err != nil { + vmCtx.Logger.Error(err, "failed to parse template", "templateStr", templateStr) + return normalizeStr(templateStr) + } + var doc bytes.Buffer + err = templ.Execute(&doc, &templateData) + if err != nil { + vmCtx.Logger.Error(err, "failed to execute template", "templateStr", templateStr) + return normalizeStr(templateStr) + } + return normalizeStr(doc.String()) + } + + return renderTemplate +} + +func v1a1TemplateFunctions( + networkStatusV1A1 v1alpha1.NetworkStatus, + networkDevicesStatusV1A1 []v1alpha1.NetworkDeviceStatus) map[string]any { + + // Get the first IP address from the first NIC. + v1alpha1FirstIP := func() (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A1[0].IPAddresses[0], nil + } + + // Get the first NIC's MAC address. + v1alpha1FirstNicMacAddr := func() (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A1[0].MacAddress, nil + } + + // Get the first IP address from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha1FirstIPFromNIC := func(index int) (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A1) { + return "", errors.New("index out of bound") + } + return networkDevicesStatusV1A1[index].IPAddresses[0], nil + } + + // Get all IP addresses from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha1IPsFromNIC := func(index int) ([]string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return []string{""}, errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A1) { + return []string{""}, errors.New("index out of bound") + } + return networkDevicesStatusV1A1[index].IPAddresses, nil + } + + // Format the first occurred count of nameservers with specific delimiter + // A negative count number would mean format all nameservers + v1alpha1FormatNameservers := func(count int, delimiter string) (string, error) { + var nameservers []string + if len(networkStatusV1A1.Nameservers) == 0 { + return "", errors.New("no available nameservers, check with VI admin") + } + if count < 0 || count >= len(networkStatusV1A1.Nameservers) { + nameservers = networkStatusV1A1.Nameservers + return strings.Join(nameservers, delimiter), nil + } + nameservers = networkStatusV1A1.Nameservers[:count] + return strings.Join(nameservers, delimiter), nil + } + + // Get subnet mask from a CIDR notation IP address and prefix length + // if IP address and prefix length not valid, throw an error and template string won't be parsed + v1alpha1SubnetMask := func(cidr string) (string, error) { + _, ipv4Net, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + netmask := fmt.Sprintf("%d.%d.%d.%d", ipv4Net.Mask[0], ipv4Net.Mask[1], ipv4Net.Mask[2], ipv4Net.Mask[3]) + return netmask, nil + } + + // Format an IP address with default netmask CIDR + // if IP not valid, throw an error and template string won't be parsed + v1alpha1IP := func(IP string) (string, error) { + if net.ParseIP(IP) == nil { + return "", errors.New("input IP address not valid") + } + defaultMask := net.ParseIP(IP).DefaultMask() + ones, _ := defaultMask.Size() + expectedCidrNotation := IP + "/" + fmt.Sprintf("%d", int32(ones)) + return expectedCidrNotation, nil + } + + // Format an IP address with network length(eg. /24) or decimal + // notation (eg. 255.255.255.0). Format an IP/CIDR with updated mask. + // An empty mask causes just the IP to be returned. + v1alpha1FormatIP := func(s string, mask string) (string, error) { + // Get the IP address for the input string. + ip, _, err := net.ParseCIDR(s) + if err != nil { + ip = net.ParseIP(s) + if ip == nil { + return "", fmt.Errorf("input IP address not valid") + } + } + // Store the IP as a string back into s. + s = ip.String() + + // If no mask was provided then return just the IP. + if mask == "" { + return s, nil + } + + // The provided mask is a network length. + if strings.HasPrefix(mask, "/") { + s += mask + if _, _, err := net.ParseCIDR(s); err != nil { + return "", err + } + return s, nil + } + + // The provided mask is subnet mask. + maskIP := net.ParseIP(mask) + if maskIP == nil { + return "", fmt.Errorf("mask is an invalid IP") + } + + maskIPBytes := maskIP.To4() + if len(maskIPBytes) == 0 { + maskIPBytes = maskIP.To16() + } + + ipNet := net.IPNet{ + IP: ip, + Mask: net.IPMask(maskIPBytes), + } + s = ipNet.String() + + // Validate the ipNet is an IP/CIDR + if _, _, err := net.ParseCIDR(s); err != nil { + return "", fmt.Errorf("invalid ip net: %s", s) + } + + return s, nil + } + + return template.FuncMap{ + constants.V1alpha1FirstIP: v1alpha1FirstIP, + constants.V1alpha1FirstNicMacAddr: v1alpha1FirstNicMacAddr, + constants.V1alpha1FirstIPFromNIC: v1alpha1FirstIPFromNIC, + constants.V1alpha1IPsFromNIC: v1alpha1IPsFromNIC, + constants.V1alpha1FormatNameservers: v1alpha1FormatNameservers, + // These are more util function that we've conflated version namespaces. + constants.V1alpha1SubnetMask: v1alpha1SubnetMask, + constants.V1alpha1IP: v1alpha1IP, + constants.V1alpha1FormatIP: v1alpha1FormatIP, + } +} + +func toTemplateNetworkStatus(bsArgs *BootstrapArgs) []v1alpha2.NetworkDeviceStatus { + networkDevicesStatus := make([]v1alpha2.NetworkDeviceStatus, 0, len(bsArgs.NetworkResults.Results)) + + for _, result := range bsArgs.NetworkResults.Results { + // When using Sysprep, the MAC address must be in the format of "-". + // CloudInit normalizes it again to ":" when adding it to the netplan. + macAddr := strings.ReplaceAll(result.MacAddress, ":", "-") + + status := v1alpha2.NetworkDeviceStatus{ + MacAddress: macAddr, + } + + for _, ipConfig := range result.IPConfigs { + // We mostly only did IPv4 before so keep that going. + if ipConfig.IsIPv4 { + if status.Gateway4 == "" { + status.Gateway4 = ipConfig.Gateway + } + + status.IPAddresses = append(status.IPAddresses, ipConfig.IPCIDR) + } + } + + networkDevicesStatus = append(networkDevicesStatus, status) + } + + return networkDevicesStatus +} + +// This is basically identical to v1a1TemplateFunctions. +func v1a2TemplateFunctions( + networkStatusV1A2 v1alpha2.NetworkStatus, + networkDevicesStatusV1A2 []v1alpha2.NetworkDeviceStatus) map[string]any { + + // Get the first IP address from the first NIC. + v1alpha2FirstIP := func() (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A2[0].IPAddresses[0], nil + } + + // Get the first NIC's MAC address. + v1alpha2FirstNicMacAddr := func() (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A2[0].MacAddress, nil + } + + // Get the first IP address from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha2FirstIPFromNIC := func(index int) (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A2) { + return "", errors.New("index out of bound") + } + return networkDevicesStatusV1A2[index].IPAddresses[0], nil + } + + // Get all IP addresses from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha2IPsFromNIC := func(index int) ([]string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return []string{""}, errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A2) { + return []string{""}, errors.New("index out of bound") + } + return networkDevicesStatusV1A2[index].IPAddresses, nil + } + + // Format the first occurred count of nameservers with specific delimiter + // A negative count number would mean format all nameservers + v1alpha2FormatNameservers := func(count int, delimiter string) (string, error) { + var nameservers []string + if len(networkStatusV1A2.Nameservers) == 0 { + return "", errors.New("no available nameservers, check with VI admin") + } + if count < 0 || count >= len(networkStatusV1A2.Nameservers) { + nameservers = networkStatusV1A2.Nameservers + return strings.Join(nameservers, delimiter), nil + } + nameservers = networkStatusV1A2.Nameservers[:count] + return strings.Join(nameservers, delimiter), nil + } + + // Get subnet mask from a CIDR notation IP address and prefix length + // if IP address and prefix length not valid, throw an error and template string won't be parsed + v1alpha2SubnetMask := func(cidr string) (string, error) { + _, ipv4Net, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + netmask := fmt.Sprintf("%d.%d.%d.%d", ipv4Net.Mask[0], ipv4Net.Mask[1], ipv4Net.Mask[2], ipv4Net.Mask[3]) + return netmask, nil + } + + // Format an IP address with default netmask CIDR + // if IP not valid, throw an error and template string won't be parsed + v1alpha2IP := func(IP string) (string, error) { + if net.ParseIP(IP) == nil { + return "", errors.New("input IP address not valid") + } + defaultMask := net.ParseIP(IP).DefaultMask() + ones, _ := defaultMask.Size() + expectedCidrNotation := IP + "/" + fmt.Sprintf("%d", int32(ones)) + return expectedCidrNotation, nil + } + + // Format an IP address with network length(eg. /24) or decimal + // notation (eg. 255.255.255.0). Format an IP/CIDR with updated mask. + // An empty mask causes just the IP to be returned. + v1alpha2FormatIP := func(s string, mask string) (string, error) { + // Get the IP address for the input string. + ip, _, err := net.ParseCIDR(s) + if err != nil { + ip = net.ParseIP(s) + if ip == nil { + return "", fmt.Errorf("input IP address not valid") + } + } + // Store the IP as a string back into s. + s = ip.String() + + // If no mask was provided then return just the IP. + if mask == "" { + return s, nil + } + + // The provided mask is a network length. + if strings.HasPrefix(mask, "/") { + s += mask + if _, _, err := net.ParseCIDR(s); err != nil { + return "", err + } + return s, nil + } + + // The provided mask is subnet mask. + maskIP := net.ParseIP(mask) + if maskIP == nil { + return "", fmt.Errorf("mask is an invalid IP") + } + + maskIPBytes := maskIP.To4() + if len(maskIPBytes) == 0 { + maskIPBytes = maskIP.To16() + } + + ipNet := net.IPNet{ + IP: ip, + Mask: net.IPMask(maskIPBytes), + } + s = ipNet.String() + + // Validate the ipNet is an IP/CIDR + if _, _, err := net.ParseCIDR(s); err != nil { + return "", fmt.Errorf("invalid ip net: %s", s) + } + + return s, nil + } + + return template.FuncMap{ + constants.V1alpha2FirstIP: v1alpha2FirstIP, + constants.V1alpha2FirstNicMacAddr: v1alpha2FirstNicMacAddr, + constants.V1alpha2FirstIPFromNIC: v1alpha2FirstIPFromNIC, + constants.V1alpha2IPsFromNIC: v1alpha2IPsFromNIC, + constants.V1alpha2FormatNameservers: v1alpha2FormatNameservers, + // These are more util function that we've conflated version namespaces. + constants.V1alpha2SubnetMask: v1alpha2SubnetMask, + constants.V1alpha2IP: v1alpha2IP, + constants.V1alpha2FormatIP: v1alpha2FormatIP, + } +} + +func toTemplateNetworkStatusV1A1(bsArgs *BootstrapArgs) []v1alpha1.NetworkDeviceStatus { + networkDevicesStatus := make([]v1alpha1.NetworkDeviceStatus, 0, len(bsArgs.NetworkResults.Results)) + + for _, result := range bsArgs.NetworkResults.Results { + // When using Sysprep, the MAC address must be in the format of "-". + // CloudInit normalizes it again to ":" when adding it to the netplan. + macAddr := strings.ReplaceAll(result.MacAddress, ":", "-") + + status := v1alpha1.NetworkDeviceStatus{ + MacAddress: macAddr, + } + + for _, ipConfig := range result.IPConfigs { + // We mostly only did IPv4 before so keep that going. + if ipConfig.IsIPv4 { + if status.Gateway4 == "" { + status.Gateway4 = ipConfig.Gateway + } + + status.IPAddresses = append(status.IPAddresses, ipConfig.IPCIDR) + } + } + + networkDevicesStatus = append(networkDevicesStatus, status) + } + + return networkDevicesStatus +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go new file mode 100644 index 000000000..18c6a6b22 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go @@ -0,0 +1,221 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("TemplateVMMetadata", func() { + + const ( + ip1 = "192.168.1.37" + ip1Cidr = ip1 + "/24" + ip2 = "192.168.10.48" + ip2Cidr = ip2 + "/24" + gateway1 = "192.168.1.1" + gateway2 = "192.168.10.1" + nameserver1 = "8.8.8.8" + nameserver2 = "1.1.1.1" + macAddr1 = "8a-cb-a0-1d-8d-c4" + macAddr2 = "00-cb-30-42-05-89" + ) + + var ( + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + bsArgs *vmlifecycle.BootstrapArgs + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-vm", + Namespace: "dummy-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithName("bootstrap-template-tests"), + VM: vm, + } + + bsArgs = &vmlifecycle.BootstrapArgs{} + bsArgs.Data = make(map[string]string) + bsArgs.DNSServers = []string{nameserver1, nameserver2} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: gateway1, + IPCIDR: ip1Cidr, + IsIPv4: true, + }, + }, + }, + { + MacAddress: macAddr2, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: gateway2, + IPCIDR: ip2Cidr, + IsIPv4: true, + }, + }, + }, + } + }) + + Context("Template Functions", func() { + DescribeTable("v1alpha1 template functions", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("first_cidrIp", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}", ip1Cidr), + Entry("second_cidrIp", "{{ (index (index .V1alpha1.Net.Devices 1).IPAddresses 0) }}", ip2Cidr), + Entry("first_gateway", "{{ (index .V1alpha1.Net.Devices 0).Gateway4 }}", gateway1), + Entry("second_gateway", "{{ (index .V1alpha1.Net.Devices 1).Gateway4 }}", gateway2), + Entry("nameserver", "{{ (index .V1alpha1.Net.Nameservers 0) }}", nameserver1), + Entry("first_macAddr", "{{ (index .V1alpha1.Net.Devices 0).MacAddress }}", macAddr1), + Entry("second_macAddr", "{{ (index .V1alpha1.Net.Devices 1).MacAddress }}", macAddr2), + Entry("name", "{{ .V1alpha1.VM.Name }}", "dummy-vm"), + ) + + DescribeTable("v1alpha2 template functions", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("first_cidrIp", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}", ip1Cidr), + Entry("second_cidrIp", "{{ (index (index .V1alpha2.Net.Devices 1).IPAddresses 0) }}", ip2Cidr), + Entry("first_gateway", "{{ (index .V1alpha2.Net.Devices 0).Gateway4 }}", gateway1), + Entry("second_gateway", "{{ (index .V1alpha2.Net.Devices 1).Gateway4 }}", gateway2), + Entry("nameserver", "{{ (index .V1alpha2.Net.Nameservers 0) }}", nameserver1), + Entry("first_macAddr", "{{ (index .V1alpha2.Net.Devices 0).MacAddress }}", macAddr1), + Entry("second_macAddr", "{{ (index .V1alpha2.Net.Devices 1).MacAddress }}", macAddr2), + Entry("name", "{{ .V1alpha2.VM.Name }}", "dummy-vm"), + ) + }) + + Context("Function names", func() { + DescribeTable("v1alpha1 constant names", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("cidr_ip1", "{{ "+constants.V1alpha1FirstIP+" }}", ip1Cidr), + Entry("cidr_ip2", "{{ "+constants.V1alpha1FirstIPFromNIC+" 1 }}", ip2Cidr), + Entry("cidr_ip3", "{{ ("+constants.V1alpha1IP+" \"192.168.1.37\") }}", ip1Cidr), + Entry("cidr_ip4", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"/24\") }}", ip1Cidr), + Entry("cidr_ip5", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip6", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip7", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"/24\") }}", ip1Cidr), + Entry("ip1", "{{ "+constants.V1alpha1FormatIP+" "+constants.V1alpha1FirstIP+" \"\" }}", ip1), + Entry("ip2", "{{ "+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"\" }}", ip1), + Entry("ips_1", "{{ "+constants.V1alpha1IPsFromNIC+" 0 }}", fmt.Sprint([]string{ip1Cidr})), + Entry("subnetmask", "{{ "+constants.V1alpha1SubnetMask+" \"192.168.1.37/26\" }}", "255.255.255.192"), + Entry("firstNicMacAddr", "{{ "+constants.V1alpha1FirstNicMacAddr+" }}", macAddr1), + Entry("formatted_nameserver1", "{{ "+constants.V1alpha1FormatNameservers+" 1 \"-\"}}", nameserver1), + Entry("formatted_nameserver2", "{{ "+constants.V1alpha1FormatNameservers+" -1 \"-\"}}", nameserver1+"-"+nameserver2), + ) + + DescribeTable("v1alpha2 constant names", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("cidr_ip1", "{{ "+constants.V1alpha2FirstIP+" }}", ip1Cidr), + Entry("cidr_ip2", "{{ "+constants.V1alpha2FirstIPFromNIC+" 1 }}", ip2Cidr), + Entry("cidr_ip3", "{{ ("+constants.V1alpha2IP+" \"192.168.1.37\") }}", ip1Cidr), + Entry("cidr_ip4", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"/24\") }}", ip1Cidr), + Entry("cidr_ip5", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip6", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip7", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"/24\") }}", ip1Cidr), + Entry("ip1", "{{ "+constants.V1alpha2FormatIP+" "+constants.V1alpha1FirstIP+" \"\" }}", ip1), + Entry("ip2", "{{ "+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"\" }}", ip1), + Entry("ips_1", "{{ "+constants.V1alpha2IPsFromNIC+" 0 }}", fmt.Sprint([]string{ip1Cidr})), + Entry("subnetmask", "{{ "+constants.V1alpha2SubnetMask+" \"192.168.1.37/26\" }}", "255.255.255.192"), + Entry("firstNicMacAddr", "{{ "+constants.V1alpha2FirstNicMacAddr+" }}", macAddr1), + Entry("formatted_nameserver1", "{{ "+constants.V1alpha2FormatNameservers+" 1 \"-\"}}", nameserver1), + Entry("formatted_nameserver2", "{{ "+constants.V1alpha2FormatNameservers+" -1 \"-\"}}", nameserver1+"-"+nameserver2), + ) + }) + + Context("Invalid template names", func() { + DescribeTable("returns the original text", + func(str string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(str)) + }, + Entry("ip1", "{{ "+constants.V1alpha1IP+" \"192.1.0\" }}"), + Entry("ip2", "{{ "+constants.V1alpha1FirstIPFromNIC+" 5 }}"), + Entry("ips_1", "{{ "+constants.V1alpha1IPsFromNIC+" 5 }}"), + Entry("cidr_ip1", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"127.255.255.255\") }}"), + Entry("cidr_ip2", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1\" \"255.0.0.0\") }}"), + Entry("gateway", "{{ (index .V1alpha1.Net.NetworkInterfaces ).Gateway }}"), + Entry("nameserver", "{{ (index .V1alpha1.Net.NameServers 0) }}"), + ) + + DescribeTable("returns the original text, v1a2 style", + func(str string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(str)) + }, + Entry("ip1", "{{ "+constants.V1alpha2IP+" \"192.1.0\" }}"), + Entry("ip2", "{{ "+constants.V1alpha2FirstIPFromNIC+" 5 }}"), + Entry("ips_1", "{{ "+constants.V1alpha2IPsFromNIC+" 5 }}"), + Entry("cidr_ip1", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"127.255.255.255\") }}"), + Entry("cidr_ip2", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1\" \"255.0.0.0\") }}"), + Entry("gateway", "{{ (index .V1alpha2.Net.NetworkInterfaces ).Gateway }}"), + Entry("nameserver", "{{ (index .V1alpha2.Net.NameServers 0) }}"), + ) + }) + + Context("String has escape characters", func() { + DescribeTable("return one level of escaped removed", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("skip_data1", "\\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data2", "\\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data3", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data4", "skip \\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "skip {{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + ) + + DescribeTable("return one level of escaped removed, v1a2 style", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("skip_data1", "\\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data2", "\\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data3", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data4", "skip \\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "skip {{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + ) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go new file mode 100644 index 000000000..11d3762cc --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go @@ -0,0 +1,51 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("Customization utils", func() { + Context("IsPending", func() { + var extraConfig []vimTypes.BaseOptionValue + var pending bool + + BeforeEach(func() { + extraConfig = nil + }) + + JustBeforeEach(func() { + pending = vmlifecycle.IsCustomizationPendingExtraConfig(extraConfig) + }) + + Context("Empty ExtraConfig", func() { + It("not pending", func() { + Expect(pending).To(BeFalse()) + }) + }) + + Context("ExtraConfig with pending key", func() { + BeforeEach(func() { + extraConfig = append(extraConfig, &vimTypes.OptionValue{ + Key: constants.GOSCPendingExtraConfigKey, + Value: "/foo/bar", + }) + }) + + It("is pending", func() { + Expect(pending).To(BeTrue()) + }) + }) + }) +}) + +// TODO: We should at least a few basic DoBootstrap() tests so we test the overall +// Reconfigure/Customize flow but the old code didn't. diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go new file mode 100644 index 000000000..241f28928 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go @@ -0,0 +1,108 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +func BootstrapVAppConfig( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + + return configSpec, nil, nil +} + +func GetOVFVAppConfigForConfigSpec( + config *vimTypes.VirtualMachineConfigInfo, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + vAppData map[string]string, + vAppExData map[string]map[string]string, + templateRenderFn TemplateRenderFunc) vimTypes.BaseVmConfigSpec { + + if config.VAppConfig == nil { + // BMV: Should we really silently return here and below? + return nil + } + + vAppConfigInfo := config.VAppConfig.GetVmConfigInfo() + if vAppConfigInfo == nil { + return nil + } + + if len(vAppConfigSpec.Properties) > 0 { + vAppData = map[string]string{} + + for _, p := range vAppConfigSpec.Properties { + if p.Value.Value != nil { + vAppData[p.Key] = *p.Value.Value + } else if p.Value.From != nil { + from := p.Value.From + vAppData[p.Key] = vAppExData[from.Name][from.Key] + } + } + } + + if templateRenderFn != nil { + // If we have a templating func, apply it to whatever data we have, regardless of the source. + for k, v := range vAppData { + vAppData[k] = templateRenderFn(k, v) + } + } + + return GetMergedvAppConfigSpec(vAppData, vAppConfigInfo.Property) +} + +// GetMergedvAppConfigSpec prepares a vApp VmConfigSpec which will set the provided key/value fields. +// Only fields marked userConfigurable and pre-existing on the VM (ie. originated from the OVF Image) +// will be set, and all others will be ignored. +func GetMergedvAppConfigSpec(inProps map[string]string, vmProps []vimTypes.VAppPropertyInfo) *vimTypes.VmConfigSpec { + outProps := make([]vimTypes.VAppPropertySpec, 0) + + for _, vmProp := range vmProps { + if vmProp.UserConfigurable == nil || !*vmProp.UserConfigurable { + continue + } + + inPropValue, found := inProps[vmProp.Id] + if !found || vmProp.Value == inPropValue { + continue + } + + vmPropCopy := vmProp + vmPropCopy.Value = inPropValue + outProp := vimTypes.VAppPropertySpec{ + ArrayUpdateSpec: vimTypes.ArrayUpdateSpec{ + Operation: vimTypes.ArrayUpdateOperationEdit, + }, + Info: &vmPropCopy, + } + outProps = append(outProps, outProp) + } + + if len(outProps) == 0 { + return nil + } + + return &vimTypes.VmConfigSpec{ + Property: outProps, + // Ensure the transport is guestInfo in case the VM does not have + // a CD-ROM device required to use the ISO transport. + OvfEnvironmentTransport: []string{OvfEnvironmentTransportGuestInfo}, + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go new file mode 100644 index 000000000..6533b3f6b --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go @@ -0,0 +1,267 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("VAppConfig Bootstrap", func() { + const key, value = "fooKey", "fooValue" + + var ( + configInfo *types.VirtualMachineConfigInfo + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + bsArgs vmlifecycle.BootstrapArgs + baseVMConfigSpec types.BaseVmConfigSpec + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + bsArgs.VAppData = make(map[string]string) + bsArgs.VAppExData = make(map[string]map[string]string) + }) + + AfterEach(func() { + vAppConfigSpec = nil + baseVMConfigSpec = nil + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("GetOVFVAppConfigForConfigSpec", func() { + + JustBeforeEach(func() { + baseVMConfigSpec = vmlifecycle.GetOVFVAppConfigForConfigSpec( + configInfo, + vAppConfigSpec, + bsArgs.VAppData, + bsArgs.VAppExData, + bsArgs.TemplateRenderFn) + }) + + Context("Empty input", func() { + It("No changes", func() { + Expect(baseVMConfigSpec).To(BeNil()) + }) + }) + + Context("vAppData Map", func() { + BeforeEach(func() { + bsArgs.VAppData[key] = value + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + + Context("vAppDataConfig Inlined Properties", func() { + BeforeEach(func() { + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + + Context("vAppDataConfig From Properties", func() { + const secretName = "my-other-secret" + + BeforeEach(func() { + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{ + From: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: key, + Optional: nil, // TODO: Rethink if we really need this complexity + }, + }, + }, + }, + } + + bsArgs.VAppExData[secretName] = map[string]string{key: value} + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + }) +}) + +var _ = Describe("GetMergedvAppConfigSpec", func() { + + DescribeTable("returns expected props", + func(inProps map[string]string, vmProps []types.VAppPropertyInfo, expected *types.VmConfigSpec) { + vAppConfigSpec := vmlifecycle.GetMergedvAppConfigSpec(inProps, vmProps) + if expected == nil { + Expect(vAppConfigSpec).To(BeNil()) + } else { + Expect(vAppConfigSpec.Property).To(HaveLen(len(expected.Property))) + for i := range vAppConfigSpec.Property { + Expect(vAppConfigSpec.Property[i].Info.Key).To(Equal(expected.Property[i].Info.Key)) + Expect(vAppConfigSpec.Property[i].Info.Id).To(Equal(expected.Property[i].Info.Id)) + Expect(vAppConfigSpec.Property[i].Info.Value).To(Equal(expected.Property[i].Info.Value)) + Expect(vAppConfigSpec.Property[i].ArrayUpdateSpec.Operation).To(Equal(types.ArrayUpdateOperationEdit)) + } + Expect(vAppConfigSpec.OvfEnvironmentTransport).To(HaveLen(1)) + Expect(vAppConfigSpec.OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + } + }, + Entry("return nil for absent vm and input props", + map[string]string{}, + []types.VAppPropertyInfo{}, + nil, + ), + Entry("return nil for non UserConfigurable vm props", + map[string]string{ + "one-id": "one-override-value", + "two-id": "two-override-value", + }, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value"}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(false)}, + }, + nil, + ), + Entry("return nil for UserConfigurable vm props but no input props", + map[string]string{}, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value"}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(true)}, + }, + nil, + ), + Entry("return valid vAppConfigSpec for setting mixed UserConfigurable props", + map[string]string{ + "one-id": "one-override-value", + "two-id": "two-override-value", + "three-id": "three-override-value", + }, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value", UserConfigurable: nil}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(true)}, + {Key: 3, Id: "three-id", Value: "three-value", UserConfigurable: pointer.Bool(false)}, + }, + &types.VmConfigSpec{ + Property: []types.VAppPropertySpec{ + {Info: &types.VAppPropertyInfo{Key: 2, Id: "two-id", Value: "two-override-value"}}, + }, + }, + ), + ) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go new file mode 100644 index 000000000..a6da98006 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go @@ -0,0 +1,41 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +// CreateArgs contains the arguments needed to create a VM. +type CreateArgs struct { + UseContentLibrary bool + ProviderItemID string + + ConfigSpec *types.VirtualMachineConfigSpec + StorageProvisioning string + FolderMoID string + ResourcePoolMoID string + HostMoID string + StorageProfileID string + DatastoreMoID string // gce2e only: used only if StorageProfileID is unset +} + +func CreateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + clClient contentlibrary.Provider, + restClient *rest.Client, + finder *find.Finder, + createArgs *CreateArgs) (*types.ManagedObjectReference, error) { + + if createArgs.UseContentLibrary { + return deployFromContentLibrary(vmCtx, clClient, restClient, createArgs) + } + + return cloneVMFromInventory(vmCtx, finder, createArgs) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go new file mode 100644 index 000000000..299d0bba7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go @@ -0,0 +1,188 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + vimtypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/placement" +) + +// CloneVMFromInventory creates a new VM by cloning the source VM. This is not reachable/used +// in production because we only really support deploying an OVF via content library. +// Maybe someday we'll use clone to speed up VM deployment so keep this code around and unit tested. +func cloneVMFromInventory( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + srcVMName := createArgs.ProviderItemID // AKA: vmCtx.VM.Spec.ImageName + + srcVM, err := finder.VirtualMachine(vmCtx, srcVMName) + if err != nil { + return nil, errors.Wrapf(err, "failed to find clone source VM: %s", srcVMName) + } + + cloneSpec, err := createCloneSpec(vmCtx, createArgs, srcVM) + if err != nil { + return nil, errors.Wrap(err, "failed to create CloneSpec") + } + + // We always set cloneSpec.Location.Folder so use that to get the parent folder object. + folder := object.NewFolder(srcVM.Client(), *cloneSpec.Location.Folder) + + cloneTask, err := srcVM.Clone(vmCtx, folder, cloneSpec.Config.Name, *cloneSpec) + if err != nil { + return nil, err + } + + result, err := cloneTask.WaitForResult(vmCtx, nil) + if err != nil { + return nil, errors.Wrapf(err, "clone VM task failed") + } + + ref := result.Result.(vimtypes.ManagedObjectReference) + return &ref, nil +} + +func createCloneSpec( + vmCtx context.VirtualMachineContextA2, + createArgs *CreateArgs, + srcVM *object.VirtualMachine) (*vimtypes.VirtualMachineCloneSpec, error) { + + cloneSpec := &vimtypes.VirtualMachineCloneSpec{ + Config: createArgs.ConfigSpec, + Memory: pointer.Bool(false), // No full memory clones. + } + + virtualDevices, err := srcVM.Device(vmCtx) + if err != nil { + return nil, fmt.Errorf("failed to get clone source VM devices: %w", err) + } + + virtualDisks := virtualDevices.SelectByType((*vimtypes.VirtualDisk)(nil)) + + for _, deviceChange := range resizeBootDiskDeviceChange(vmCtx, virtualDisks) { + if deviceChange.GetVirtualDeviceConfigSpec().Operation == vimtypes.VirtualDeviceConfigSpecOperationEdit { + cloneSpec.Location.DeviceChange = append(cloneSpec.Location.DeviceChange, deviceChange) + } else { + cloneSpec.Config.DeviceChange = append(cloneSpec.Config.DeviceChange, deviceChange) + } + } + + if createArgs.StorageProfileID != "" { + cloneSpec.Location.Profile = []vimtypes.BaseVirtualMachineProfileSpec{ + &vimtypes.VirtualMachineDefinedProfileSpec{ProfileId: createArgs.StorageProfileID}, + } + } else { + // BMV: Used to compute placement? Otherwise, always overwritten later. + cloneSpec.Location.Datastore = &vimtypes.ManagedObjectReference{ + Type: "Datastore", + Value: createArgs.DatastoreMoID, + } + } + + cloneSpec.Location.Pool = &vimtypes.ManagedObjectReference{ + Type: "ResourcePool", + Value: createArgs.ResourcePoolMoID, + } + cloneSpec.Location.Folder = &vimtypes.ManagedObjectReference{ + Type: "Folder", + Value: createArgs.FolderMoID, + } + + rpOwner, err := object.NewResourcePool(srcVM.Client(), *cloneSpec.Location.Pool).Owner(vmCtx) + if err != nil { + return nil, err + } + + cluster, ok := rpOwner.(*object.ClusterComputeResource) + if !ok { + return nil, fmt.Errorf("owner of the ResourcePool is not a cluster but %T", rpOwner) + } + + relocateSpec, err := placement.CloneVMRelocateSpec(vmCtx, cluster, srcVM.Reference(), cloneSpec) + if err != nil { + return nil, err + } + + cloneSpec.Location.Host = relocateSpec.Host + cloneSpec.Location.Datastore = relocateSpec.Datastore + cloneSpec.Location.Disk = cloneVMDiskLocators(virtualDisks, createArgs, cloneSpec.Location) + + return cloneSpec, nil +} + +func cloneVMDiskLocators( + disks object.VirtualDeviceList, + createArgs *CreateArgs, + location vimtypes.VirtualMachineRelocateSpec) []vimtypes.VirtualMachineRelocateSpecDiskLocator { + + diskLocators := make([]vimtypes.VirtualMachineRelocateSpecDiskLocator, 0, len(disks)) + + for _, disk := range disks { + locator := vimtypes.VirtualMachineRelocateSpecDiskLocator{ + DiskId: disk.GetVirtualDevice().Key, + Datastore: *location.Datastore, + Profile: location.Profile, + // TODO: Check if policy is encrypted and use correct DiskMoveType + DiskMoveType: string(vimtypes.VirtualMachineRelocateDiskMoveOptionsMoveChildMostDiskBacking), + } + + if backing, ok := disk.(*vimtypes.VirtualDisk).Backing.(*vimtypes.VirtualDiskFlatVer2BackingInfo); ok { + switch createArgs.StorageProvisioning { + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeThin): + backing.ThinProvisioned = pointer.Bool(true) + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeThick): + backing.ThinProvisioned = pointer.Bool(false) + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeEagerZeroedThick): + backing.EagerlyScrub = pointer.Bool(true) + } + locator.DiskBackingInfo = backing + } + + diskLocators = append(diskLocators, locator) + } + + return diskLocators +} + +func resizeBootDiskDeviceChange( + vmCtx context.VirtualMachineContextA2, + virtualDisks object.VirtualDeviceList) []vimtypes.BaseVirtualDeviceConfigSpec { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil + } + + // Assume the first virtual disk - if any - is the boot disk. + var deviceChanges []vimtypes.BaseVirtualDeviceConfigSpec + for _, vmDevice := range virtualDisks { + vmDisk, ok := vmDevice.(*vimtypes.VirtualDisk) + if !ok { + continue + } + + // Maybe don't allow shrink? + if vmDisk.CapacityInBytes != capacity.Value() { + vmDisk.CapacityInBytes = capacity.Value() + deviceChanges = append(deviceChanges, &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationEdit, + Device: vmDisk, + }) + } + + break + } + + return deviceChanges +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go new file mode 100644 index 000000000..e0edefb55 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go @@ -0,0 +1,105 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "encoding/base64" + "fmt" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vapi/vcenter" + vimtypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +func deployOVF( + vmCtx context.VirtualMachineContextA2, + restClient *rest.Client, + item *library.Item, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + deploymentSpec := vcenter.DeploymentSpec{ + Name: vmCtx.VM.Name, + StorageProfileID: createArgs.StorageProfileID, + StorageProvisioning: createArgs.StorageProvisioning, + AcceptAllEULA: true, + } + + if deploymentSpec.StorageProfileID == "" { + // Without a storage profile, fall back to the datastore. + deploymentSpec.DefaultDatastoreID = createArgs.DatastoreMoID + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && createArgs.ConfigSpec != nil { + configSpecXML, err := util.MarshalConfigSpecToXML(createArgs.ConfigSpec) + if err != nil { + return nil, fmt.Errorf("failed to marshal ConfigSpec to XML: %w", err) + } + + deploymentSpec.VmConfigSpec = &vcenter.VmConfigSpec{ + Provider: constants.ConfigSpecProviderXML, + XML: base64.StdEncoding.EncodeToString(configSpecXML), + } + } + + deploy := vcenter.Deploy{ + DeploymentSpec: deploymentSpec, + Target: vcenter.Target{ + ResourcePoolID: createArgs.ResourcePoolMoID, + FolderID: createArgs.FolderMoID, + HostID: createArgs.HostMoID, + }, + } + + vmCtx.Logger.Info("Deploying OVF Library Item", "itemID", item.ID, "itemName", item.Name, "deploy", deploy) + + return vcenter.NewManager(restClient).DeployLibraryItem(vmCtx, item.ID, deploy) +} + +func deployVMTX( + vmCtx context.VirtualMachineContextA2, + restClient *rest.Client, + item *library.Item, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + // Not yet supported. This item type needs to be deployed via DeployTemplateLibraryItem(), + // which doesn't take a ConfigSpec so it is a heavy lift. + // TODO: We should catch this earlier to avoid a bunch of wasted work. + + _ = vmCtx + _ = restClient + _ = createArgs + + return nil, fmt.Errorf("creating VM from VMTX content library type is not supported: %s", item.Name) +} + +func deployFromContentLibrary( + vmCtx context.VirtualMachineContextA2, + clClient contentlibrary.Provider, + restClient *rest.Client, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + // This call is needed to get the item type. We could avoid going to CL here, and + // instead get the item type via the {Cluster}ContentLibrary CR for the image. + item, err := clClient.GetLibraryItemID(vmCtx, createArgs.ProviderItemID) + if err != nil { + return nil, err + } + + switch item.Type { + case library.ItemTypeOVF: + return deployOVF(vmCtx, restClient, item, createArgs) + case library.ItemTypeVMTX: + return deployVMTX(vmCtx, restClient, item, createArgs) + default: + return nil, errors.Errorf("item %s not a supported type: %s", item.Name, item.Type) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go new file mode 100644 index 000000000..62bf35ba4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go @@ -0,0 +1,345 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + "net" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var ( + // The minimum properties needed to be retrieved in order to populate the Status. Callers may + // provide a MO with more. This often saves us a second round trip in the common steady state. + vmStatusPropertiesSelector = []string{"config.changeTrackingEnabled", "guest", "summary"} +) + +func UpdateStatus( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vcVM *object.VirtualMachine, + vmMO *mo.VirtualMachine) error { + + vm := vmCtx.VM + + // This is implicitly true: ensure the condition is set since it is how we determine the old v1a1 Phase. + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionCreated) + // TODO: Might set other "prereq" conditions too for version conversion but we'd have to fib a little. + + if vm.Status.Image == nil { + // If unset, we don't know if this was a cluster or namespace scoped image at the time. + vm.Status.Image = &common.LocalObjectRef{Name: vm.Spec.ImageName} + } + if vm.Status.Class == nil { + // We can most likely just assume the other fields of the LocalObjectRef but only fill + // in the Name for now. Our handling of this field will be more complicated once we really + // support class changes and reconfiguring the VM the fly in response. + vm.Status.Class = &common.LocalObjectRef{Name: vm.Spec.ClassName} + } + + if vmMO == nil { + // In the common case, our caller will have already gotten the MO properties in order to determine + // if it had any reconciliation to do, and there was nothing to do since the VM is in the steady + // state so that MO is still entirely valid here. + // NOTE: The properties must have been retrieved with at least vmStatusPropertiesSelector. + vmMO = &mo.VirtualMachine{} + if err := vcVM.Properties(vmCtx, vcVM.Reference(), vmStatusPropertiesSelector, vmMO); err != nil { + // Leave the current Status unchanged for now. + return fmt.Errorf("failed to get VM properties for status update: %w", err) + } + } + + var errs []error + var err error + summary := vmMO.Summary + + vm.Status.PowerState = convertPowerState(summary.Runtime.PowerState) + vm.Status.UniqueID = vcVM.Reference().Value + vm.Status.BiosUUID = summary.Config.Uuid + vm.Status.InstanceUUID = summary.Config.InstanceUuid + vm.Status.Network = getGuestNetworkStatus(vmMO.Guest) + + vm.Status.Host, err = getRuntimeHostHostname(vmCtx, vcVM, summary.Runtime.Host) + if err != nil { + errs = append(errs, err) + } + + MarkVMToolsRunningStatusCondition(vmCtx.VM, vmMO.Guest) + MarkCustomizationInfoCondition(vmCtx.VM, vmMO.Guest) + + if config := vmMO.Config; config != nil { + vm.Status.ChangeBlockTracking = config.ChangeTrackingEnabled + } else { + vm.Status.ChangeBlockTracking = nil + } + + if lib.IsWcpFaultDomainsFSSEnabled() { + zoneName := vm.Labels[topology.KubernetesTopologyZoneLabelKey] + if zoneName == "" { + cluster, err := virtualmachine.GetVMClusterComputeResource(vmCtx, vcVM) + if err != nil { + errs = append(errs, err) + } else { + zoneName, err = topology.LookupZoneForClusterMoID(vmCtx, k8sClient, cluster.Reference().Value) + if err != nil { + errs = append(errs, err) + } else { + if vm.Labels == nil { + vm.Labels = map[string]string{} + } + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + } + } + } + + if zoneName != "" { + vm.Status.Zone = zoneName + } + } + + return k8serrors.NewAggregate(errs) +} + +func getRuntimeHostHostname( + ctx goctx.Context, + vcVM *object.VirtualMachine, + host *types.ManagedObjectReference) (string, error) { + + if host != nil { + return object.NewHostSystem(vcVM.Client(), *host).ObjectName(ctx) + } + return "", nil +} + +func getGuestNetworkStatus(guestInfo *types.GuestInfo) *vmopv1.VirtualMachineNetworkStatus { + if guestInfo == nil { + return nil + } + + status := &vmopv1.VirtualMachineNetworkStatus{} + + if ipAddr := guestInfo.IpAddress; ipAddr != "" { + // TODO: Filter out local addresses. + if net.ParseIP(ipAddr).To4() != nil { + status.PrimaryIP4 = ipAddr + } else { + status.PrimaryIP6 = ipAddr + } + } + + if len(guestInfo.Net) > 0 { + status.Interfaces = make([]vmopv1.VirtualMachineNetworkInterfaceStatus, 0, len(guestInfo.Net)) + for i := range guestInfo.Net { + status.Interfaces = append(status.Interfaces, guestNicInfoToInterfaceStatus(i, &guestInfo.Net[i])) + } + } + + if len(guestInfo.IpStack) > 0 { + status.VirtualMachineNetworkIPStackStatus = guestIPStackInfoToIPStackStatus(&guestInfo.IpStack[0]) + } + + return status +} + +func guestNicInfoToInterfaceStatus(idx int, guestNicInfo *types.GuestNicInfo) vmopv1.VirtualMachineNetworkInterfaceStatus { + status := vmopv1.VirtualMachineNetworkInterfaceStatus{} + + // TODO: What name exactly? The CRD name may be the most useful here but hard to line that up. + // BMV: DeviceConfigId will be -1 for our pseudo-y interfaces. Most likely want to just skip those devices. + status.Name = fmt.Sprintf("nic-%d-%d", idx, guestNicInfo.DeviceConfigId) + status.IP.MACAddr = guestNicInfo.MacAddress + + if guestIPConfig := guestNicInfo.IpConfig; guestIPConfig != nil { + ip := &status.IP + + ip.AutoConfigurationEnabled = guestIPConfig.AutoConfigurationEnabled + ip.Addresses = convertNetIPConfigInfoIPAddresses(guestIPConfig.IpAddress) + + if guestIPConfig.Dhcp != nil { + ip.DHCP = convertNetDhcpConfigInfo(guestIPConfig.Dhcp) + } + } + + if dnsConfig := guestNicInfo.DnsConfig; dnsConfig != nil { + status.DNS = convertNetDNSConfigInfo(dnsConfig) + } + + return status +} + +func guestIPStackInfoToIPStackStatus(guestIPStack *types.GuestStackInfo) vmopv1.VirtualMachineNetworkIPStackStatus { + status := vmopv1.VirtualMachineNetworkIPStackStatus{} + + if dhcpConfig := guestIPStack.DhcpConfig; dhcpConfig != nil { + status.DHCP = convertNetDhcpConfigInfo(dhcpConfig) + } + + if dnsConfig := guestIPStack.DnsConfig; dnsConfig != nil { + status.DNS = convertNetDNSConfigInfo(dnsConfig) + } + + if ipRouteConfig := guestIPStack.IpRouteConfig; ipRouteConfig != nil { + status.IPRoutes = convertNetIPRouteConfigInfo(ipRouteConfig) + } + + status.KernelConfig = convertKeyValueSlice(guestIPStack.IpStackConfig) + + return status +} + +func convertPowerState(powerState types.VirtualMachinePowerState) vmopv1.VirtualMachinePowerState { + switch powerState { + case types.VirtualMachinePowerStatePoweredOff: + return vmopv1.VirtualMachinePowerStateOff + case types.VirtualMachinePowerStatePoweredOn: + return vmopv1.VirtualMachinePowerStateOn + case types.VirtualMachinePowerStateSuspended: + return vmopv1.VirtualMachinePowerStateSuspended + } + return "" +} + +func convertNetIPConfigInfoIPAddresses(ipAddresses []types.NetIpConfigInfoIpAddress) []vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus { + if len(ipAddresses) == 0 { + return nil + } + + out := make([]vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus, 0, len(ipAddresses)) + for _, guestIPAddr := range ipAddresses { + ipAddrStatus := vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus{ + Address: guestIPAddr.IpAddress, + Origin: guestIPAddr.Origin, + State: guestIPAddr.State, + } + if guestIPAddr.Lifetime != nil { + ipAddrStatus.Lifetime = metav1.NewTime(*guestIPAddr.Lifetime) + } + + out = append(out, ipAddrStatus) + } + return out +} + +func convertNetDNSConfigInfo(dnsConfig *types.NetDnsConfigInfo) vmopv1.VirtualMachineNetworkDNSStatus { + return vmopv1.VirtualMachineNetworkDNSStatus{ + DHCP: dnsConfig.Dhcp, + DomainName: dnsConfig.DomainName, + HostName: dnsConfig.HostName, + Nameservers: dnsConfig.IpAddress, + SearchDomains: dnsConfig.SearchDomain, + } +} + +func convertNetDhcpConfigInfo(dhcpConfig *types.NetDhcpConfigInfo) vmopv1.VirtualMachineNetworkDHCPStatus { + status := vmopv1.VirtualMachineNetworkDHCPStatus{} + + if ipv4 := dhcpConfig.Ipv4; ipv4 != nil { + status.IP4.Enabled = ipv4.Enable + status.IP4.Config = convertKeyValueSlice(ipv4.Config) + } + + if ipv6 := dhcpConfig.Ipv6; ipv6 != nil { + status.IP6.Enabled = ipv6.Enable + status.IP6.Config = convertKeyValueSlice(ipv6.Config) + } + + return status +} + +func convertNetIPRouteConfigInfo(routeConfig *types.NetIpRouteConfigInfo) []vmopv1.VirtualMachineNetworkIPRouteStatus { + if len(routeConfig.IpRoute) == 0 { + return nil + } + + // TODO: Prob only want to show default routes. Will be very verbose on TKG nodes. + out := make([]vmopv1.VirtualMachineNetworkIPRouteStatus, 0, len(routeConfig.IpRoute)) + for _, ipRoute := range routeConfig.IpRoute { + out = append(out, vmopv1.VirtualMachineNetworkIPRouteStatus{ + Gateway: vmopv1.VirtualMachineNetworkIPRouteGatewayStatus{ + Device: ipRoute.Gateway.Device, + Address: ipRoute.Gateway.IpAddress, + }, + NetworkAddress: fmt.Sprintf("%s/%d", ipRoute.Network, ipRoute.PrefixLength), + }) + } + return out +} + +func convertKeyValueSlice(s []types.KeyValue) []common.KeyValuePair { + if len(s) == 0 { + return nil + } + + out := make([]common.KeyValuePair, 0, len(s)) + for i := range s { + out = append(out, common.KeyValuePair{Key: s[i].Key, Value: s[i].Value}) + } + return out +} + +func MarkVMToolsRunningStatusCondition( + vm *vmopv1.VirtualMachine, + guestInfo *types.GuestInfo) { + + if guestInfo == nil || guestInfo.ToolsRunningStatus == "" { + conditions.MarkUnknown(vm, vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", "") + return + } + + switch guestInfo.ToolsRunningStatus { + case string(types.VirtualMachineToolsRunningStatusGuestToolsNotRunning): + msg := "VMware Tools is not running" + conditions.MarkFalse(vm, vmopv1.VirtualMachineToolsCondition, vmopv1.VirtualMachineToolsNotRunningReason, msg) + case string(types.VirtualMachineToolsRunningStatusGuestToolsRunning), string(types.VirtualMachineToolsRunningStatusGuestToolsExecutingScripts): + conditions.MarkTrue(vm, vmopv1.VirtualMachineToolsCondition) + default: + msg := "Unexpected VMware Tools running status" + conditions.MarkUnknown(vm, vmopv1.VirtualMachineToolsCondition, "Unknown", msg) + } +} + +func MarkCustomizationInfoCondition(vm *vmopv1.VirtualMachine, guestInfo *types.GuestInfo) { + if guestInfo == nil || guestInfo.CustomizationInfo == nil { + conditions.MarkUnknown(vm, vmopv1.GuestCustomizationCondition, "NoGuestInfo", "") + return + } + + switch guestInfo.CustomizationInfo.CustomizationStatus { + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_IDLE), "": + conditions.MarkTrue(vm, vmopv1.GuestCustomizationCondition) + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_PENDING): + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationPendingReason, "") + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_RUNNING): + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationRunningReason, "") + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_SUCCEEDED): + conditions.MarkTrue(vm, vmopv1.GuestCustomizationCondition) + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_FAILED): + errorMsg := guestInfo.CustomizationInfo.ErrorMsg + if errorMsg == "" { + errorMsg = "vSphere VM Customization failed due to an unknown error." + } + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationFailedReason, errorMsg) + default: + errorMsg := guestInfo.CustomizationInfo.ErrorMsg + if errorMsg == "" { + errorMsg = "Unexpected VM Customization status" + } + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, "Unknown", errorMsg) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go new file mode 100644 index 000000000..3d130cdc9 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("VirtualMachineTools Status to VM Status Condition", func() { + Context("markVMToolsRunningStatusCondition", func() { + var ( + vm *vmopv1.VirtualMachine + guestInfo *types.GuestInfo + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{} + guestInfo = &types.GuestInfo{ + ToolsRunningStatus: "", + } + }) + + JustBeforeEach(func() { + vmlifecycle.MarkVMToolsRunningStatusCondition(vm, guestInfo) + }) + + Context("guestInfo is nil", func() { + BeforeEach(func() { + guestInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("ToolsRunningStatus is empty", func() { + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is not running", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsNotRunning) + }) + It("sets condition to false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineToolsCondition, vmopv1.VirtualMachineToolsNotRunningReason, "VMware Tools is not running"), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is running", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsRunning) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.VirtualMachineToolsCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is starting", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsExecutingScripts) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.VirtualMachineToolsCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("Unexpected vmtools running status", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = "blah" + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "Unknown", "Unexpected VMware Tools running status"), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + }) +}) + +var _ = Describe("VSphere Customization Status to VM Status Condition", func() { + Context("markCustomizationInfoCondition", func() { + var ( + vm *vmopv1.VirtualMachine + guestInfo *types.GuestInfo + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{} + guestInfo = &types.GuestInfo{ + CustomizationInfo: &types.GuestInfoCustomizationInfo{}, + } + }) + + JustBeforeEach(func() { + vmlifecycle.MarkCustomizationInfoCondition(vm, guestInfo) + }) + + Context("guestInfo unset", func() { + BeforeEach(func() { + guestInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.GuestCustomizationCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo unset", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.GuestCustomizationCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo idle", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_IDLE) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.GuestCustomizationCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo pending", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_PENDING) + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationPendingReason, ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo running", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_RUNNING) + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationRunningReason, ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo succeeded", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_SUCCEEDED) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.GuestCustomizationCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo failed", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_FAILED) + guestInfo.CustomizationInfo.ErrorMsg = "some error message" + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationFailedReason, guestInfo.CustomizationInfo.ErrorMsg), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo invalid", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = "asdf" + guestInfo.CustomizationInfo.ErrorMsg = "some error message" + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, "Unknown", guestInfo.CustomizationInfo.ErrorMsg), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go new file mode 100644 index 000000000..461db366f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestVMLifecycle(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider VM Lifecycle Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider.go b/pkg/vmprovider/providers/vsphere2/vmprovider.go new file mode 100644 index 000000000..161b24262 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider.go @@ -0,0 +1,428 @@ +// Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + goctx "context" + "encoding/json" + "fmt" + "math/rand" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/task" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/record" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + vcconfig "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +const ( + VsphereVMProviderName = "vsphere" + + // taskHistoryCollectorPageSize represents the max count to read from task manager in one iteration. + taskHistoryCollectorPageSize = 10 + + ovfCacheMaxItem = 100 + ovfCacheItemExpiration = 30 * time.Minute + ovfCacheExpirationCheckInterval = 5 * time.Minute +) + +var log = logf.Log.WithName(VsphereVMProviderName) + +type VersionedOVFEnvelope struct { + OvfEnvelope *ovf.Envelope + ContentVersion string +} + +type vSphereVMProvider struct { + k8sClient ctrlruntime.Client + eventRecorder record.Recorder + globalExtraConfig map[string]string + minCPUFreq uint64 + ovfCache *util.Cache[VersionedOVFEnvelope] + ovfCacheLockPool *util.LockPool[string, *sync.RWMutex] + + vcClientLock sync.Mutex + vcClient *vcclient.Client +} + +func NewVSphereVMProviderFromClient( + client ctrlruntime.Client, + recorder record.Recorder) vmprovider.VirtualMachineProviderInterfaceA2 { + + ovfCache, ovfLockPool := InitOvfCacheAndLockPool( + ovfCacheItemExpiration, ovfCacheExpirationCheckInterval, ovfCacheMaxItem) + + return &vSphereVMProvider{ + k8sClient: client, + eventRecorder: recorder, + globalExtraConfig: getExtraConfig(), + ovfCache: ovfCache, + ovfCacheLockPool: ovfLockPool, + } +} + +// InitOvfCacheAndLockPool initializes the ovf cache and lock pool that are used +// to cache the ovf envelope and lock the ovf envelope when it is being downloaded. +func InitOvfCacheAndLockPool(expireAfter, checkExpireInterval time.Duration, maxItems int) ( + *util.Cache[VersionedOVFEnvelope], *util.LockPool[string, *sync.RWMutex]) { + ovfCache := util.NewCache[VersionedOVFEnvelope](expireAfter, checkExpireInterval, maxItems) + ovfLockPool := &util.LockPool[string, *sync.RWMutex]{} + + // Clean up the lock pool when the ovf cache item expires. + expiredChan := ovfCache.ExpiredChan() + go func() { + for k := range expiredChan { + l := ovfLockPool.Get(k) + // This could still delete an in-use lock if it's retrieved from the pool but not locked yet. + // If it's already locked, this will wait until it's unlocked to delete it from the pool. + l.Lock() + ovfLockPool.Delete(k) + l.Unlock() + } + }() + + return ovfCache, ovfLockPool +} + +func getExtraConfig() map[string]string { + ec := map[string]string{ + constants.EnableDiskUUIDExtraConfigKey: constants.ExtraConfigTrue, + constants.GOSCIgnoreToolsCheckExtraConfigKey: constants.ExtraConfigTrue, + } + + if jsonEC := os.Getenv("JSON_EXTRA_CONFIG"); jsonEC != "" { + extraConfig := make(map[string]string) + + if err := json.Unmarshal([]byte(jsonEC), &extraConfig); err != nil { + // This is only set in testing so make errors fatal. + panic(fmt.Sprintf("invalid JSON_EXTRA_CONFIG envvar: %q %v", jsonEC, err)) + } + + for k, v := range extraConfig { + ec[k] = v + } + } + + return ec +} + +func (vs *vSphereVMProvider) getVcClient(ctx goctx.Context) (*vcclient.Client, error) { + vs.vcClientLock.Lock() + defer vs.vcClientLock.Unlock() + + if vs.vcClient != nil { + return vs.vcClient, nil + } + + config, err := vcconfig.GetProviderConfig(ctx, vs.k8sClient) + if err != nil { + return nil, err + } + + vcClient, err := vcclient.NewClient(ctx, config) + if err != nil { + return nil, err + } + + vs.vcClient = vcClient + return vcClient, nil +} + +func (vs *vSphereVMProvider) UpdateVcPNID(ctx goctx.Context, vcPNID, vcPort string) error { + updated, err := vcconfig.UpdateVcInConfigMap(ctx, vs.k8sClient, vcPNID, vcPort) + if err != nil || !updated { + return err + } + + // Our controller-runtime client does not cache ConfigMaps & Secrets, so the next time + // getVcClient() is called, it will fetch newly updated CM. + vs.clearAndLogoutVcClient(ctx) + return nil +} + +func (vs *vSphereVMProvider) ResetVcClient(ctx goctx.Context) { + vs.clearAndLogoutVcClient(ctx) +} + +func (vs *vSphereVMProvider) clearAndLogoutVcClient(ctx goctx.Context) { + vs.vcClientLock.Lock() + vcClient := vs.vcClient + vs.vcClient = nil + vs.vcClientLock.Unlock() + + if vcClient != nil { + vcClient.Logout(ctx) + } +} + +// SyncVirtualMachineImage syncs the vmi object with the OVF Envelope retrieved from the cli object. +func (vs *vSphereVMProvider) SyncVirtualMachineImage(ctx goctx.Context, cli, vmi ctrlruntime.Object) error { + var itemID, contentVersion string + switch cli := cli.(type) { + case *imgregv1a1.ContentLibraryItem: + itemID = string(cli.Spec.UUID) + contentVersion = cli.Status.ContentVersion + case *imgregv1a1.ClusterContentLibraryItem: + itemID = string(cli.Spec.UUID) + contentVersion = cli.Status.ContentVersion + default: + return errors.Errorf("unexpected content library item type %T", cli) + } + + ovfEnvelope, err := vs.getOvfEnvelope(ctx, itemID, contentVersion) + if err != nil { + return err + } + + logger := log.V(4).WithValues("vmiName", vmi.GetName(), "cliName", cli.GetName()) + if ovfEnvelope == nil { + logger.Error(nil, "skip syncing VMI as corresponding OVF envelope is nil") + return nil + } + + contentlibrary.UpdateVmiWithOvfEnvelope(vmi, *ovfEnvelope) + return nil +} + +// getOvfEnvelope gets the OVF envelope from the cache if it exists and matches version. +// If not, it downloads the OVF envelope from vCenter and stores it in the cache. +func (vs *vSphereVMProvider) getOvfEnvelope( + ctx goctx.Context, itemID, contentVersion string) (*ovf.Envelope, error) { + logger := log.V(4).WithValues("itemID", itemID, "contentVersion", contentVersion) + + // Lock the current item to prevent concurrent downloads of the same OVF. + // This is done before the get from cache below to prevent stale result. + curItemLock := vs.ovfCacheLockPool.Get(itemID) + curItemLock.Lock() + defer curItemLock.Unlock() + + isHitFunc := func(cacheItem VersionedOVFEnvelope) bool { + return cacheItem.ContentVersion == contentVersion + } + cacheItem, found := vs.ovfCache.Get(itemID, isHitFunc) + if found { + logger.Info("Cache item hit, using cached OVF") + } else { + logger.Info("Cache item miss, downloading OVF from vCenter") + client, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + ovfEnvelope, err := client.ContentLibClient().RetrieveOvfEnvelopeByLibraryItemID(ctx, itemID) + if err != nil || ovfEnvelope == nil { + return nil, err + } + + cacheItem = VersionedOVFEnvelope{ + ContentVersion: contentVersion, + OvfEnvelope: ovfEnvelope, + } + putResult := vs.ovfCache.Put(itemID, cacheItem) + logger.Info("Cache item put", "itemID", itemID, "putResult", putResult) + } + + return cacheItem.OvfEnvelope, nil +} + +// GetItemFromLibraryByName get the library item from specified content library by its name. +// Do not return error if the item doesn't exist in the content library. +func (vs *vSphereVMProvider) GetItemFromLibraryByName(ctx goctx.Context, + contentLibrary, itemName string) (*library.Item, error) { + log.V(4).Info("Get item from ContentLibrary", + "UUID", contentLibrary, "item name", itemName) + + client, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + return client.ContentLibClient().GetLibraryItem(ctx, contentLibrary, itemName, false) +} + +func (vs *vSphereVMProvider) UpdateContentLibraryItem(ctx goctx.Context, itemID, newName string, newDescription *string) error { + log.V(4).Info("Update Content Library Item", "itemID", itemID) + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + return client.ContentLibClient().UpdateLibraryItem(ctx, itemID, newName, newDescription) +} + +func (vs *vSphereVMProvider) getOpID(vm *vmopv1.VirtualMachine, operation string) string { + const charset = "0123456789abcdef" + + id := make([]byte, 8) + for i := range id { + idx := rand.Intn(len(charset)) //nolint:gosec + id[i] = charset[idx] + } + + return strings.Join([]string{"vmoperator", vm.Name, operation, string(id)}, "-") +} + +func (vs *vSphereVMProvider) getVM( + vmCtx context.VirtualMachineContextA2, + client *vcclient.Client, + notFoundReturnErr bool) (*object.VirtualMachine, error) { + + vcVM, err := vcenter.GetVirtualMachine(vmCtx, vs.k8sClient, client.VimClient(), client.Datacenter(), client.Finder()) + if err != nil { + return nil, err + } + + if vcVM == nil && notFoundReturnErr { + return nil, fmt.Errorf("VirtualMachine %q was not found on VC", vmCtx.VM.Name) + } + + return vcVM, nil +} + +func (vs *vSphereVMProvider) getOrComputeCPUMinFrequency(ctx goctx.Context) (uint64, error) { + minFreq := atomic.LoadUint64(&vs.minCPUFreq) + if minFreq == 0 { + // The infra controller hasn't finished ComputeCPUMinFrequency() yet, so try to + // compute that value now. + var err error + minFreq, err = vs.computeCPUMinFrequency(ctx) + if err != nil { + // minFreq may be non-zero in case of partial success. + return minFreq, err + } + + // Update value if not updated already. + atomic.CompareAndSwapUint64(&vs.minCPUFreq, 0, minFreq) + } + + return minFreq, nil +} + +func (vs *vSphereVMProvider) ComputeCPUMinFrequency(ctx goctx.Context) error { + minFreq, err := vs.computeCPUMinFrequency(ctx) + if err != nil { + // Might have a partial success (non-zero freq): store that if we haven't updated + // the min freq yet, and let the controller retry. This whole min CPU freq thing + // is kind of unfortunate & busted. + atomic.CompareAndSwapUint64(&vs.minCPUFreq, 0, minFreq) + return err + } + + atomic.StoreUint64(&vs.minCPUFreq, minFreq) + return nil +} + +func (vs *vSphereVMProvider) computeCPUMinFrequency(ctx goctx.Context) (uint64, error) { + // Get all the availability zones in order to calculate the minimum + // CPU frequencies for each of the zones' vSphere clusters. + availabilityZones, err := topology.GetAvailabilityZones(ctx, vs.k8sClient) + if err != nil { + return 0, err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return 0, err + } + + if !lib.IsWcpFaultDomainsFSSEnabled() { + ccr, err := vcenter.GetResourcePoolOwnerMoRef(ctx, client.VimClient(), client.Config().ResourcePool) + if err != nil { + return 0, err + } + + // Only expect 1 AZ in this case. + for i := range availabilityZones { + availabilityZones[i].Spec.ClusterComputeResourceMoIDs = []string{ccr.Value} + } + } + + var errs []error + + var minFreq uint64 + for _, az := range availabilityZones { + moIDs := az.Spec.ClusterComputeResourceMoIDs + if len(moIDs) == 0 { + moIDs = []string{az.Spec.ClusterComputeResourceMoId} // HA TEMP + } + + for _, moID := range moIDs { + ccr := object.NewClusterComputeResource(client.VimClient(), + types.ManagedObjectReference{Type: "ClusterComputeResource", Value: moID}) + + freq, err := vcenter.ClusterMinCPUFreq(ctx, ccr) + if err != nil { + errs = append(errs, err) + } else if minFreq == 0 || freq < minFreq { + minFreq = freq + } + } + } + + return minFreq, k8serrors.NewAggregate(errs) +} + +func (vs *vSphereVMProvider) GetTasksByActID(ctx goctx.Context, actID string) (_ []types.TaskInfo, retErr error) { + vcClient, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + taskManager := task.NewManager(vcClient.VimClient()) + filterSpec := types.TaskFilterSpec{ + ActivationId: []string{actID}, + } + + collector, err := taskManager.CreateCollectorForTasks(ctx, filterSpec) + if err != nil { + return nil, errors.Wrapf(err, "failed to create collector for tasks") + } + defer func() { + err = collector.Destroy(ctx) + if retErr == nil { + retErr = err + } + }() + + taskList := make([]types.TaskInfo, 0) + for { + nextTasks, err := collector.ReadNextTasks(ctx, taskHistoryCollectorPageSize) + if err != nil { + log.Error(err, "failed to read next tasks") + return nil, err + } + if len(nextTasks) == 0 { + break + } + taskList = append(taskList, nextTasks...) + } + + log.V(5).Info("found tasks", "actID", actID, "tasks", taskList) + return taskList, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go new file mode 100644 index 000000000..aae8db2f4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go @@ -0,0 +1,261 @@ +// Copyright (c) 2020-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + "context" + "fmt" + + vimtypes "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +// IsVirtualMachineSetResourcePolicyReady checks if the VirtualMachineSetResourcePolicy for the AZ is ready. +func (vs *vSphereVMProvider) IsVirtualMachineSetResourcePolicyReady( + ctx context.Context, + azName string, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (bool, error) { + + client, err := vs.getVcClient(ctx) + if err != nil { + return false, err + } + + folderMoID, rpMoID, err := topology.GetNamespaceFolderAndRPMoID(ctx, vs.k8sClient, azName, resourcePolicy.Namespace) + if err != nil { + return false, err + } + + folderExists, err := vcenter.DoesChildFolderExist(ctx, client.VimClient(), folderMoID, resourcePolicy.Spec.Folder) + if err != nil { + return false, err + } + + rpExists, err := vcenter.DoesChildResourcePoolExist(ctx, client.VimClient(), rpMoID, resourcePolicy.Spec.ResourcePool.Name) + if err != nil { + return false, err + } + + clusterRef, err := vcenter.GetResourcePoolOwnerMoRef(ctx, client.VimClient(), rpMoID) + if err != nil { + return false, err + } + + modulesExist, err := vs.doClusterModulesExist(ctx, client.ClusterModuleClient(), clusterRef.Reference(), resourcePolicy) + if err != nil { + return false, err + } + + if !rpExists || !folderExists || !modulesExist { + log.V(4).Info("Resource policy is not ready", "resourcePolicy", resourcePolicy.Name, + "namespace", resourcePolicy.Name, "az", azName, "resourcePool", rpExists, "folder", folderExists, "modules", modulesExist) + return false, nil + } + + return true, nil +} + +// CreateOrUpdateVirtualMachineSetResourcePolicy creates if a VirtualMachineSetResourcePolicy doesn't exist, updates otherwise. +func (vs *vSphereVMProvider) CreateOrUpdateVirtualMachineSetResourcePolicy( + ctx context.Context, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + folderMoID, rpMoIDs, err := vs.getNamespaceFolderAndRPMoIDs(ctx, resourcePolicy.Namespace) + if err != nil { + return err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + vimClient := client.VimClient() + var errs []error + + _, err = vcenter.CreateFolder(ctx, vimClient, folderMoID, resourcePolicy.Spec.Folder) + if err != nil { + errs = append(errs, err) + } + + for _, rpMoID := range rpMoIDs { + _, err := vcenter.CreateOrUpdateChildResourcePool(ctx, vimClient, rpMoID, &resourcePolicy.Spec.ResourcePool) + if err != nil { + errs = append(errs, err) + } + + clusterRef, err := vcenter.GetResourcePoolOwnerMoRef(ctx, vimClient, rpMoID) + if err == nil { + err = vs.createClusterModules(ctx, client.ClusterModuleClient(), clusterRef.Reference(), resourcePolicy) + } + if err != nil { + errs = append(errs, err) + } + } + + return k8serrors.NewAggregate(errs) +} + +// DeleteVirtualMachineSetResourcePolicy deletes the VirtualMachineSetPolicy. +func (vs *vSphereVMProvider) DeleteVirtualMachineSetResourcePolicy( + ctx context.Context, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + folderMoID, rpMoIDs, err := vs.getNamespaceFolderAndRPMoIDs(ctx, resourcePolicy.Namespace) + if err != nil { + return err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + vimClient := client.VimClient() + var errs []error + + for _, rpMoID := range rpMoIDs { + err := vcenter.DeleteChildResourcePool(ctx, vimClient, rpMoID, resourcePolicy.Spec.ResourcePool.Name) + if err != nil { + errs = append(errs, err) + } + } + + errs = append(errs, vs.deleteClusterModules(ctx, client.ClusterModuleClient(), resourcePolicy)...) + + if err := vcenter.DeleteChildFolder(ctx, vimClient, folderMoID, resourcePolicy.Spec.Folder); err != nil { + errs = append(errs, err) + } + + return k8serrors.NewAggregate(errs) +} + +// doClusterModulesExist checks whether all the ClusterModules for the given VirtualMachineSetResourcePolicy +// have been created and exist in VC for the Session's Cluster. +func (vs *vSphereVMProvider) doClusterModulesExist( + ctx context.Context, + clusterModProvider clustermodules.Provider, + clusterRef vimtypes.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (bool, error) { + + for _, groupName := range resourcePolicy.Spec.ClusterModuleGroups { + _, moduleID := clustermodules.FindClusterModuleUUID(groupName, clusterRef, resourcePolicy) + if moduleID == "" { + return false, nil + } + + exists, err := clusterModProvider.DoesModuleExist(ctx, moduleID, clusterRef) + if !exists || err != nil { + return false, err + } + } + + return true, nil +} + +// createClusterModules creates all the ClusterModules that has not created yet for a +// given VirtualMachineSetResourcePolicy in VC. +func (vs *vSphereVMProvider) createClusterModules( + ctx context.Context, + clusterModProvider clustermodules.Provider, + clusterRef vimtypes.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + var errs []error + + // There is no way to give a name when creating a VC cluster module, so we have to + // resort to using the status as the source of truth. This can result in orphaned + // modules if, for instance, we fail to update the resource policy k8s object. + for _, groupName := range resourcePolicy.Spec.ClusterModuleGroups { + idx, moduleID := clustermodules.FindClusterModuleUUID(groupName, clusterRef, resourcePolicy) + + if moduleID != "" { + // Verify this cluster module exists on VC for this cluster. + exists, err := clusterModProvider.DoesModuleExist(ctx, moduleID, clusterRef) + if err != nil { + errs = append(errs, err) + continue + } + if !exists { + // Status entry is stale. Create below. + moduleID = "" + } + } else { + var err error + // See if there is already a module for this cluster but without the ClusterMoID field + // set that we can claim. + idx, moduleID, err = clustermodules.ClaimClusterModuleUUID(ctx, clusterModProvider, + groupName, clusterRef, resourcePolicy) + if err != nil { + errs = append(errs, err) + continue + } + } + + if moduleID == "" { + var err error + moduleID, err = clusterModProvider.CreateModule(ctx, clusterRef) + if err != nil { + errs = append(errs, err) + continue + } + } + + if idx >= 0 { + resourcePolicy.Status.ClusterModules[idx].ModuleUuid = moduleID + resourcePolicy.Status.ClusterModules[idx].ClusterMoID = clusterRef.Value + } else { + status := vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName, + ModuleUuid: moduleID, + ClusterMoID: clusterRef.Value, + } + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, status) + } + } + + return k8serrors.NewAggregate(errs) +} + +// deleteClusterModules deletes all the ClusterModules associated with a given VirtualMachineSetResourcePolicy in VC. +func (vs *vSphereVMProvider) deleteClusterModules( + ctx context.Context, + clusterModProvider clustermodules.Provider, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) []error { + + var errModStatus []vmopv1.VSphereClusterModuleStatus + var errs []error + + for _, moduleStatus := range resourcePolicy.Status.ClusterModules { + err := clusterModProvider.DeleteModule(ctx, moduleStatus.ModuleUuid) + if err != nil { + errModStatus = append(errModStatus, moduleStatus) + errs = append(errs, err) + } + } + + resourcePolicy.Status.ClusterModules = errModStatus + return errs +} + +func (vs *vSphereVMProvider) getNamespaceFolderAndRPMoIDs( + ctx context.Context, + namespace string) (string, []string, error) { + + folderMoID, rpMoIDs, err := topology.GetNamespaceFolderAndRPMoIDs(ctx, vs.k8sClient, namespace) + if err != nil { + return "", nil, err + } + + if folderMoID == "" { + return "", nil, fmt.Errorf("namespace %s not present in any AvailabilityZones", namespace) + } + + return folderMoID, rpMoIDs, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go new file mode 100644 index 000000000..03c76f704 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go @@ -0,0 +1,234 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "fmt" + "path" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func getVirtualMachineSetResourcePolicy(name, namespace string) *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-resourcepolicy", name), + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: fmt.Sprintf("%s-resourcepool", name), + Reservations: vmopv1.VirtualMachineResourceSpec{}, + Limits: vmopv1.VirtualMachineResourceSpec{}, + }, + Folder: fmt.Sprintf("%s-folder", name), + ClusterModuleGroups: []string{"ControlPlane", "NodeGroup1"}, + }, + } +} + +func resourcePolicyTests() { + Describe("VirtualMachineSetResourcePolicy Tests", func() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + testConfig builder.VCSimTestConfig + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("VirtualMachineSetResourcePolicy", func() { + var ( + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + JustBeforeEach(func() { + testPolicyName := "test-policy" + + resourcePolicy = getVirtualMachineSetResourcePolicy(testPolicyName, nsInfo.Namespace) + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + }) + + JustAfterEach(func() { + Expect(vmProvider.DeleteVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).Should(BeEmpty()) + }) + + It("creates expected cluster modules", func() { + modules := resourcePolicy.Status.ClusterModules + Expect(modules).Should(HaveLen(2)) + module := modules[0] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[0])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + module = modules[1] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[1])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + }) + + Context("for an existing resource policy", func() { + It("should keep existing cluster modules", func() { + status := resourcePolicy.Status.DeepCopy() + + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).To(ContainElements(status.ClusterModules)) + }) + + It("successfully able to find the resource policy", func() { + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, "", resourcePolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + + Context("for an absent resource policy", func() { + It("should fail to find the resource policy without any errors", func() { + failResPolicy := getVirtualMachineSetResourcePolicy("test-policy", nsInfo.Namespace) + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, "", failResPolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("for a resource policy with invalid cluster module", func() { + It("successfully able to delete the resource policy", func() { + resourcePolicy.Status.ClusterModules = append([]vmopv1.VSphereClusterModuleStatus{ + { + GroupName: "invalid-group", + ModuleUuid: "invalid-uuid", + }, + }, resourcePolicy.Status.ClusterModules...) + }) + }) + + It("creates expected resource pool", func() { + rp, err := ctx.GetSingleClusterCompute().ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Make trip through the Finder to populate InventoryPath. + objRef, err := ctx.Finder.ObjectReference(ctx, rp.Reference()) + Expect(err).ToNot(HaveOccurred()) + rp, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + + inventoryPath := path.Join(rp.InventoryPath, nsInfo.Namespace, resourcePolicy.Spec.ResourcePool.Name) + _, err = ctx.Finder.ResourcePool(ctx, inventoryPath) + Expect(err).ToNot(HaveOccurred()) + }) + + It("creates expected child folder", func() { + _, err := ctx.Finder.Folder(ctx, path.Join(nsInfo.Folder.InventoryPath, resourcePolicy.Spec.Folder)) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("when HA is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + It("creates expected cluster modules for each cluster", func() { + moduleCount := len(resourcePolicy.Spec.ClusterModuleGroups) + Expect(moduleCount).To(Equal(2)) + + modules := resourcePolicy.Status.ClusterModules + zoneNames := ctx.ZoneNames + Expect(modules).To(HaveLen(len(zoneNames) * ctx.ClustersPerZone * moduleCount)) + + for zoneIdx, zoneName := range zoneNames { + // NOTE: This assumes some ordering but is the easiest way to test. + moduleIdx := zoneIdx * ctx.ClustersPerZone * moduleCount + modules := modules[moduleIdx : moduleIdx+ctx.ClustersPerZone*moduleCount] + + ccrs := ctx.GetAZClusterComputes(zoneName) + Expect(ccrs).To(HaveLen(ctx.ClustersPerZone)) + + for _, cluster := range ccrs { + clusterMoID := cluster.Reference().Value + + module := modules[0] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[0])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + Expect(module.ClusterMoID).To(Equal(clusterMoID)) + + module = modules[1] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[1])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + Expect(module.ClusterMoID).To(Equal(clusterMoID)) + } + } + }) + + It("should claim cluster module without ClusterMoID set", func() { + Expect(resourcePolicy.Spec.ClusterModuleGroups).ToNot(BeEmpty()) + groupName := resourcePolicy.Spec.ClusterModuleGroups[0] + + moduleStatus := resourcePolicy.Status.DeepCopy() + Expect(moduleStatus.ClusterModules).ToNot(BeEmpty()) + + for i := range resourcePolicy.Status.ClusterModules { + if resourcePolicy.Status.ClusterModules[i].GroupName == groupName { + resourcePolicy.Status.ClusterModules[i].ClusterMoID = "" + } + } + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).To(Equal(moduleStatus.ClusterModules)) + }) + + It("successfully able to find the resource policy in each zone", func() { + for _, zoneName := range ctx.ZoneNames { + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, zoneName, resourcePolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + } + }) + + It("creates expected resource pool for each cluster", func() { + for _, zoneName := range ctx.ZoneNames { + for _, cluster := range ctx.GetAZClusterComputes(zoneName) { + rp, err := cluster.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Make trip through the Finder to populate InventoryPath. + objRef, err := ctx.Finder.ObjectReference(ctx, rp.Reference()) + Expect(err).ToNot(HaveOccurred()) + rp, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + + inventoryPath := path.Join(rp.InventoryPath, nsInfo.Namespace, resourcePolicy.Spec.ResourcePool.Name) + _, err = ctx.Finder.ResourcePool(ctx, inventoryPath) + Expect(err).ToNot(HaveOccurred()) + } + } + }) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_test.go new file mode 100644 index 000000000..5a447c69d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "sync" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func cpuFreqTests() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterface + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + vmProvider = nil + }) + + Context("ComputeCPUMinFrequency", func() { + It("returns success", func() { + Expect(vmProvider.ComputeCPUMinFrequency(ctx)).To(Succeed()) + }) + }) +} + +func initOvfCacheAndLockPoolTests() { + + var ( + expireAfter = 3 * time.Second + checkExpireInterval = 1 * time.Second + maxItems = 3 + + ovfCache *util.Cache[vsphere.VersionedOVFEnvelope] + ovfLockPool *util.LockPool[string, *sync.RWMutex] + ) + + BeforeEach(func() { + ovfCache, ovfLockPool = vsphere.InitOvfCacheAndLockPool( + expireAfter, checkExpireInterval, maxItems) + }) + + AfterEach(func() { + ovfCache = nil + ovfLockPool = nil + }) + + Context("InitOvfCacheAndLockPool", func() { + It("should clean up lock pool when the item is expired in cache", func() { + Expect(ovfCache).ToNot(BeNil()) + Expect(ovfLockPool).ToNot(BeNil()) + + itemID := "test-item-id" + res := ovfCache.Put(itemID, vsphere.VersionedOVFEnvelope{}) + Expect(res).To(Equal(util.CachePutResultCreate)) + curItemLock := ovfLockPool.Get(itemID) + Expect(curItemLock).ToNot(BeNil()) + + Eventually(func() bool { + // ovfLockPool.Get() returns a new lock if the item key is not found. + // So the lock should be different when the item is expired and deleted from pool. + return ovfLockPool.Get(itemID) != curItemLock + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go new file mode 100644 index 000000000..fb7721812 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -0,0 +1,1169 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + goctx "context" + "fmt" + "strings" + "sync" + "text/template" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/placement" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/storage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +// VMCreateArgs contains the arguments needed to create a VM on VC. +type VMCreateArgs struct { + vmlifecycle.CreateArgs + vmlifecycle.BootstrapData + + VMClass *vmopv1.VirtualMachineClass + ResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ImageObj ctrlclient.Object + ImageSpec *vmopv1.VirtualMachineImageSpec + ImageStatus *vmopv1.VirtualMachineImageStatus + + StorageClassesToIDs map[string]string + HasInstanceStorage bool + ChildResourcePoolName string + ChildFolderName string + ClusterMoRef types.ManagedObjectReference + + NetworkResults network.NetworkInterfaceResults +} + +// TODO: Until we sort out what the Session becomes. +type vmUpdateArgs = session.VMUpdateArgs + +const ( + FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" +) + +var ( + createCountLock sync.Mutex + concurrentCreateCount int + + // SkipVMImageCLProviderCheck skips the checks that a VM Image has a Content Library item provider + // since a VirtualMachineImage created for a VM template won't have either. This has been broken for + // a long time but was otherwise masked on how the tests used to be organized. + SkipVMImageCLProviderCheck = false +) + +func (vs *vSphereVMProvider) CreateOrUpdateVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) error { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "createOrUpdateVM")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return err + } + + vcVM, err := vs.getVM(vmCtx, client, false) + if err != nil { + return err + } + + if vcVM == nil { + var createArgs *VMCreateArgs + + vcVM, createArgs, err = vs.createVirtualMachine(vmCtx, client) + if err != nil { + return err + } + + if vcVM == nil { + // Creation was not ready or blocked for some reason. We depend on the controller + // to eventually retry the create. + return nil + } + + return vs.createdVirtualMachineFallthroughUpdate(vmCtx, vcVM, client, createArgs) + } + + return vs.updateVirtualMachine(vmCtx, vcVM, client, nil) +} + +func (vs *vSphereVMProvider) DeleteVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) error { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "deleteVM")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return err + } + + vcVM, err := vs.getVM(vmCtx, client, false) + if err != nil { + return err + } else if vcVM == nil { + // VM does not exist. + return nil + } + + return virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) +} + +func (vs *vSphereVMProvider) PublishVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine, + vmPub *vmopv1.VirtualMachinePublishRequest, + cl *imgregv1a1.ContentLibrary, + actID string) (string, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: ctx, + // Update logger info + Logger: log.WithValues("vmName", vm.NamespacedName()). + WithValues("clName", fmt.Sprintf("%s/%s", cl.Namespace, cl.Name)). + WithValues("vmPubName", fmt.Sprintf("%s/%s", vmPub.Namespace, vmPub.Name)), + VM: vm, + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return "", errors.Wrapf(err, "failed to get vCenter client") + } + + itemID, err := virtualmachine.CreateOVF(vmCtx, client.RestClient(), vmPub, cl, actID) + if err != nil { + return "", err + } + + return itemID, nil +} + +// BackupVirtualMachine backs up the VM data required for restore. +func (vs *vSphereVMProvider) BackupVirtualMachine(ctx goctx.Context, vm *vmopv1.VirtualMachine) error { + // TODO + return nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineGuestHeartbeat( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "heartbeat")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return "", err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return "", err + } + + status, err := virtualmachine.GetGuestHeartBeatStatus(vmCtx, vcVM) + if err != nil { + return "", err + } + + return status, nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineWebMKSTicket( + ctx goctx.Context, + vm *vmopv1.VirtualMachine, + pubKey string) (string, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "webconsole")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return "", err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return "", err + } + + ticket, err := virtualmachine.GetWebConsoleTicket(vmCtx, vcVM, pubKey) + if err != nil { + return "", err + } + + return ticket, nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineHardwareVersion( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) (int32, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "hardware-version")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return 0, err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return 0, err + } + + var o mo.VirtualMachine + err = vcVM.Properties(vmCtx, vcVM.Reference(), []string{"config.version"}, &o) + if err != nil { + return 0, err + } + + return contentlibrary.ParseVirtualHardwareVersion(o.Config.Version), nil +} + +func (vs *vSphereVMProvider) createVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*object.VirtualMachine, *VMCreateArgs, error) { + + createArgs, err := vs.vmCreateGetArgs(vmCtx, vcClient) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateDoPlacement(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateGetFolderAndRPMoIDs(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateFixupConfigSpec(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateIsReady(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + // BMV: This is about where we used to do this check but it prb make more sense to do + // earlier, as to limit wasted work. Before DoPlacement() is likely the best place so + // the window between the placement decision and creating the VM on VC is small(ish). + allowed, createDeferFn, err := vs.vmCreateConcurrentAllowed(vmCtx) + if err != nil { + return nil, nil, err + } else if !allowed { + return nil, nil, nil + } + defer createDeferFn() + + moRef, err := vmlifecycle.CreateVirtualMachine( + vmCtx, + vcClient.ContentLibClient(), + vcClient.RestClient(), + vcClient.Finder(), + &createArgs.CreateArgs) + if err != nil { + vmCtx.Logger.Error(err, "CreateVirtualMachine failed") + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionCreated, "Error", err.Error()) + return nil, nil, err + } + + vmCtx.VM.Status.UniqueID = moRef.Reference().Value + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionCreated) + + return object.NewVirtualMachine(vcClient.VimClient(), *moRef), createArgs, nil +} + +func (vs *vSphereVMProvider) createdVirtualMachineFallthroughUpdate( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + // TODO: In the common case, we'll call directly into update right after create succeeds, and + // can use the createArgs to avoid doing a bunch of lookup work again. + + return vs.updateVirtualMachine(vmCtx, vcVM, vcClient, createArgs) +} + +func (vs *vSphereVMProvider) updateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + vmCtx.Logger.V(4).Info("Updating VirtualMachine") + + { + // Hack - create just enough of the Session that's needed for update + + cluster, err := virtualmachine.GetVMClusterComputeResource(vmCtx, vcVM) + if err != nil { + return err + } + + ses := &session.Session{ + K8sClient: vs.k8sClient, + Client: vcClient, + Finder: vcClient.Finder(), + Cluster: cluster, + } + + getUpdateArgsFn := func() (*vmUpdateArgs, error) { + // TODO: Use createArgs if we already got them + _ = createArgs + return vs.vmUpdateGetArgs(vmCtx) + } + + err = ses.UpdateVirtualMachine(vmCtx, vcVM, getUpdateArgsFn) + if err != nil { + return err + } + } + + return nil +} + +// vmCreateDoPlacement determines placement of the VM prior to creating the VM on VC. +func (vs *vSphereVMProvider) vmCreateDoPlacement( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + placementConfigSpec := virtualmachine.CreateConfigSpecForPlacement( + vmCtx, + createArgs.ConfigSpec, + createArgs.StorageClassesToIDs) + + result, err := placement.Placement( + vmCtx, + vs.k8sClient, + vcClient.VimClient(), + placementConfigSpec, + createArgs.ChildResourcePoolName) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady, "NotReady", err.Error()) + return err + } + + if result.PoolMoRef.Value != "" { + createArgs.ResourcePoolMoID = result.PoolMoRef.Value + } + + if result.HostMoRef != nil { + createArgs.HostMoID = result.HostMoRef.Value + } + + if result.InstanceStoragePlacement { + hostMoID := createArgs.HostMoID + + if hostMoID == "" { + return fmt.Errorf("placement result missing host required for instance storage") + } + + hostFQDN, err := vcenter.GetESXHostFQDN(vmCtx, vcClient.VimClient(), hostMoID) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady, "NotReady", err.Error()) + return err + } + + if vmCtx.VM.Annotations == nil { + vmCtx.VM.Annotations = map[string]string{} + } + vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey] = hostMoID + vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeAnnotationKey] = hostFQDN + } + + if result.ZonePlacement { + if vmCtx.VM.Labels == nil { + vmCtx.VM.Labels = map[string]string{} + } + // Note if the VM create fails for some reason, but this label gets updated on the k8s VM, + // then this is the pre-assigned zone on later create attempts. + vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey] = result.ZoneName + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady) + + return nil +} + +// vmCreateGetFolderAndRPMoIDs gets the MoIDs of the Folder and Resource Pool the VM will be created under. +func (vs *vSphereVMProvider) vmCreateGetFolderAndRPMoIDs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if createArgs.ResourcePoolMoID == "" { + // We did not do placement so find this namespace/zone ResourcePool and Folder. + + nsFolderMoID, rpMoID, err := topology.GetNamespaceFolderAndRPMoID(vmCtx, vs.k8sClient, + vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey], vmCtx.VM.Namespace) + if err != nil { + return err + } + + // If this VM has a ResourcePolicy ResourcePool, lookup the child ResourcePool under the + // namespace/zone's root ResourcePool. This will be the VM's ResourcePool. + if createArgs.ChildResourcePoolName != "" { + parentRP := object.NewResourcePool(vcClient.VimClient(), + types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + childRP, err := vcenter.GetChildResourcePool(vmCtx, parentRP, createArgs.ChildResourcePoolName) + if err != nil { + return err + } + + rpMoID = childRP.Reference().Value + } + + createArgs.ResourcePoolMoID = rpMoID + createArgs.FolderMoID = nsFolderMoID + + } else { + // Placement already selected the ResourcePool/Cluster, so we just need this namespace's Folder. + nsFolderMoID, err := topology.GetNamespaceFolderMoID(vmCtx, vs.k8sClient, vmCtx.VM.Namespace) + if err != nil { + return err + } + + createArgs.FolderMoID = nsFolderMoID + } + + // If this VM has a ResourcePolicy Folder, lookup the child Folder under the namespace's Folder. + // This will be the VM's parent Folder in the VC inventory. + if createArgs.ChildFolderName != "" { + parentFolder := object.NewFolder(vcClient.VimClient(), + types.ManagedObjectReference{Type: "Folder", Value: createArgs.FolderMoID}) + + childFolder, err := vcenter.GetChildFolder(vmCtx, parentFolder, createArgs.ChildFolderName) + if err != nil { + return err + } + + createArgs.FolderMoID = childFolder.Reference().Value + } + + // Now that we know the ResourcePool, use that to look up the CCR. + clusterMoRef, err := vcenter.GetResourcePoolOwnerMoRef(vmCtx, vcClient.VimClient(), createArgs.ResourcePoolMoID) + if err != nil { + return err + } + createArgs.ClusterMoRef = clusterMoRef + + return nil +} + +func (vs *vSphereVMProvider) vmCreateFixupConfigSpec( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + fixedUp, err := network.ResolveBackingPostPlacement( + vmCtx, + vcClient.VimClient(), + createArgs.ClusterMoRef, + &createArgs.NetworkResults) + if err != nil { + return err + } + + if fixedUp { + // Now that the backing is resolved for this CCR, re-zip to update the ConfigSpec. What a mess. + err = vs.vmCreateGenConfigSpecZipNetworkInterfaces(vmCtx, createArgs) + if err != nil { + return err + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateIsReady( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if policy := createArgs.ResourcePolicy; policy != nil { + // TODO: May want to do this as to filter the placement candidates. + exists, err := vs.doClusterModulesExist(vmCtx, vcClient.ClusterModuleClient(), createArgs.ClusterMoRef, policy) + if err != nil { + return err + } else if !exists { + return fmt.Errorf("VirtualMachineSetResourcePolicy cluster module is not ready") + } + } + + if createArgs.HasInstanceStorage { + if _, ok := vmCtx.VM.Annotations[constants.InstanceStoragePVCsBoundAnnotationKey]; !ok { + return fmt.Errorf("instance storage PVCs are not bound yet") + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateConcurrentAllowed(vmCtx context.VirtualMachineContextA2) (bool, func(), error) { + maxDeployThreads, ok := vmCtx.Value(context.MaxDeployThreadsContextKey).(int) + if !ok { + return false, nil, fmt.Errorf("MaxDeployThreadsContextKey missing from context") + } + + createCountLock.Lock() + if concurrentCreateCount >= maxDeployThreads { + createCountLock.Unlock() + vmCtx.Logger.Info("Too many create VirtualMachine already occurring. Re-queueing request") + return false, nil, nil + } + + concurrentCreateCount++ + createCountLock.Unlock() + + decrementFn := func() { + createCountLock.Lock() + concurrentCreateCount-- + createCountLock.Unlock() + } + + return true, decrementFn, nil +} + +func (vs *vSphereVMProvider) vmCreateGetArgs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*VMCreateArgs, error) { + + createArgs, err := vs.vmCreateGetPrereqs(vmCtx, vcClient) + if err != nil { + return nil, err + } + + err = vs.vmCreateDoNetworking(vmCtx, vcClient, createArgs) + if err != nil { + return nil, err + } + + err = vs.vmCreateGenConfigSpec(vmCtx, createArgs) + if err != nil { + return nil, err + } + + err = vs.vmCreateValidateArgs(vmCtx, vcClient, createArgs) + if err != nil { + return nil, err + } + + return createArgs, nil +} + +// vmCreateGetPrereqs returns the VMCreateArgs populated with the k8s objects required to +// create the VM on VC. +func (vs *vSphereVMProvider) vmCreateGetPrereqs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*VMCreateArgs, error) { + + createArgs := &VMCreateArgs{} + var prereqErrs []error + + if err := vs.vmCreateGetVirtualMachineClass(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetVirtualMachineImage(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetSetResourcePolicy(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetBootstrap(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetStoragePrereqs(vmCtx, vcClient, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + // This is about the point where historically we'd declare the prereqs ready or not. There + // is still a lot of work to do - and things to fail - before the actual create, but there + // is no point in continuing if the above checks aren't met since we are missing data + // required to create the VM. + if len(prereqErrs) > 0 { + return nil, k8serrors.NewAggregate(prereqErrs) + } + + // Note that once the VM is created, it is hard for us to later resolve what image was used, + // since a NS or cluster scoped image could have been created or deleted. + vmCtx.VM.Status.Image = &common.LocalObjectRef{ + APIVersion: createArgs.ImageObj.GetObjectKind().GroupVersionKind().Version, + Kind: createArgs.ImageObj.GetObjectKind().GroupVersionKind().Kind, + Name: createArgs.ImageObj.GetName(), + } + + vmCtx.VM.Status.Class = &common.LocalObjectRef{ + APIVersion: createArgs.VMClass.APIVersion, + Kind: createArgs.VMClass.Kind, + Name: createArgs.VMClass.Name, + } + + return createArgs, nil +} + +func (vs *vSphereVMProvider) vmCreateGetVirtualMachineClass( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + vmClass, err := GetVirtualMachineClass(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.VMClass = vmClass + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetVirtualMachineImage( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + imageObj, imageSpec, imageStatus, err := GetVirtualMachineImageSpecAndStatus(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.ImageObj = imageObj + createArgs.ImageSpec = imageSpec + createArgs.ImageStatus = imageStatus + + // This is clunky, but we need to know how to use the image to create the VM. Our only supported + // method is via the ContentLibrary, so check if this image was derived from a CL item. + switch imageSpec.ProviderRef.Kind { + case "ClusterContentLibraryItem", "ContentLibraryItem": + createArgs.UseContentLibrary = true + createArgs.ProviderItemID = imageStatus.ProviderItemID + default: + if !SkipVMImageCLProviderCheck { + err := fmt.Errorf("unsupported image provider kind: %s", imageSpec.ProviderRef.Kind) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, "NotSupported", err.Error()) + return err + } + // Testing only: we'll clone the source VM found in the Inventory. + createArgs.UseContentLibrary = false + createArgs.ProviderItemID = vmCtx.VM.Spec.ImageName + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetSetResourcePolicy( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + resourcePolicy, err := GetVMSetResourcePolicy(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + // The SetResourcePolicy is optional (TKG VMs will always have it). + if resourcePolicy != nil { + createArgs.ResourcePolicy = resourcePolicy + createArgs.ChildFolderName = resourcePolicy.Spec.Folder + createArgs.ChildResourcePoolName = resourcePolicy.Spec.ResourcePool.Name + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetBootstrap( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + data, vAppData, vAppExData, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.BootstrapData.Data = data + createArgs.BootstrapData.VAppData = vAppData + createArgs.BootstrapData.VAppExData = vAppExData + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetStoragePrereqs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if lib.IsInstanceStorageFSSEnabled() { + // To determine all the storage profiles, we need the class because of the possibility of + // InstanceStorage volumes. If we weren't able to get the class earlier, still check & set + // the storage condition because instance storage usage is rare, it is helpful to report + // as many prereqs as possible, and we'll reevaluate this once the class is available. + if createArgs.VMClass != nil { + // Add the class's instance storage disks - if any - to the VM.Spec. Once the instance + // storage disks are added to the VM, they are set in stone even if the class itself or + // the VM's assigned class changes. + createArgs.HasInstanceStorage = AddInstanceStorageVolumes(vmCtx, createArgs.VMClass) + } + } + + storageClassesToIDs, err := storage.GetVMStoragePoliciesIDs(vmCtx, vs.k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady, reason, msg) + return err + } + + vmStorageProfileID := storageClassesToIDs[vmCtx.VM.Spec.StorageClass] + + provisioningType, err := virtualmachine.GetDefaultDiskProvisioningType(vmCtx, vcClient, vmStorageProfileID) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady, reason, msg) + return err + } + + createArgs.StorageClassesToIDs = storageClassesToIDs + createArgs.StorageProvisioning = provisioningType + createArgs.StorageProfileID = vmStorageProfileID + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateDoNetworking( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + networkSpec := &vmCtx.VM.Spec.Network + if networkSpec.Disabled { + // No connected networking for this VM. Any EthCards will be removed later. + return nil + } + + interfaces := networkSpec.Interfaces + if len(interfaces) == 0 { + // VM gets one automatic NIC. Create the default interface from fields in the network spec. + defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: networkSpec.DeviceName, + Addresses: networkSpec.Addresses, + DHCP4: networkSpec.DHCP4, + DHCP6: networkSpec.DHCP6, + Gateway4: networkSpec.Gateway4, + Gateway6: networkSpec.Gateway6, + MTU: networkSpec.MTU, + Nameservers: networkSpec.Nameservers, + Routes: networkSpec.Routes, + SearchDomains: networkSpec.SearchDomains, + } + + if defaultInterface.Name == "" { + defaultInterface.Name = "eth0" + } + if networkSpec.Network != nil { + defaultInterface.Network = *networkSpec.Network + } + + interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} + } + + results, err := network.CreateAndWaitForNetworkInterfaces( + vmCtx, + vs.k8sClient, + vcClient.VimClient(), + vcClient.Finder(), + nil, // Don't know the CCR yet (needed to resolve backings for NSX-T) + interfaces) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionNetworkReady, "NotReady", err.Error()) + return err + } + + createArgs.NetworkResults = results + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionNetworkReady) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpec( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + // TODO: This is a partial dupe of what's done in the update path in the remaining Session code. I got + // tired of trying to keep that in sync so we get to live with a frankenstein thing longer. + + var vmClassConfigSpec *types.VirtualMachineConfigSpec + if rawConfigSpec := createArgs.VMClass.Spec.ConfigSpec; lib.IsVMClassAsConfigFSSDaynDateEnabled() && len(rawConfigSpec) > 0 { + configSpec, err := GetVMClassConfigSpec(rawConfigSpec) + if err != nil { + return err + } + vmClassConfigSpec = configSpec + } else { + vmClassConfigSpec = virtualmachine.ConfigSpecFromVMClassDevices(&createArgs.VMClass.Spec) + } + + var minCPUFreq uint64 + if res := createArgs.VMClass.Spec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + freq, err := vs.getOrComputeCPUMinFrequency(vmCtx) + if err != nil { + return err + } + minCPUFreq = freq + } + + createArgs.ConfigSpec = virtualmachine.CreateConfigSpec( + vmCtx, + vmClassConfigSpec, + &createArgs.VMClass.Spec, + createArgs.ImageStatus, + minCPUFreq) + + // TODO: This should be in CreateConfigSpec() + if createArgs.ConfigSpec.Version == "" { + imageVer := int32(0) + if createArgs.ImageStatus.HardwareVersion != nil { + imageVer = *createArgs.ImageStatus.HardwareVersion + } + + version := HardwareVersionForPVCandPCIDevices(imageVer, createArgs.ConfigSpec, HasPVC(vmCtx.VM.Spec)) + if version != 0 { + createArgs.ConfigSpec.Version = fmt.Sprintf("vmx-%d", version) + } + } + + err := vs.vmCreateGenConfigSpecExtraConfig(vmCtx, createArgs) + if err != nil { + return err + } + + err = vs.vmCreateGenConfigSpecChangeBootDiskSize(vmCtx, createArgs) + if err != nil { + return err + } + + err = vs.vmCreateGenConfigSpecZipNetworkInterfaces(vmCtx, createArgs) + if err != nil { + return err + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecExtraConfig( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + ecMap := make(map[string]string, len(vs.globalExtraConfig)) + + // The only use of this template is for the JSON_EXTRA_CONFIG that is set in gce2e env + // to populate {{.ImageName }} so vcsim will create a container for the VM. + // BMV: This should be removable now that vcsim gce2e is gone. + renderTemplateFn := func(name, text string) string { + t, err := template.New(name).Parse(text) + if err != nil { + return text + } + b := strings.Builder{} + if err := t.Execute(&b, createArgs.ImageStatus); err != nil { + return text + } + return b.String() + } + for k, v := range vs.globalExtraConfig { + ecMap[k] = renderTemplateFn(k, v) + } + + hasPassthroughDevices := len(util.SelectVirtualPCIPassthrough(util.DevicesFromConfigSpec(createArgs.ConfigSpec))) > 0 + + if hasPassthroughDevices || createArgs.HasInstanceStorage { + ecMap[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + } + + if hasPassthroughDevices { + mmioSize := vmCtx.VM.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] + if mmioSize == "" { + mmioSize = constants.PCIPassthruMMIOSizeDefault + } + if mmioSize != "0" { + ecMap[constants.PCIPassthruMMIOExtraConfigKey] = constants.ExtraConfigTrue + ecMap[constants.PCIPassthruMMIOSizeExtraConfigKey] = mmioSize + } + } + + // The ConfigSpec's current ExtraConfig values (that came from the class) take precedence over what was set here. + createArgs.ConfigSpec.ExtraConfig = util.AppendNewExtraConfigValues(createArgs.ConfigSpec.ExtraConfig, ecMap) + + // Leave constants.VMOperatorV1Alpha1ExtraConfigKey for the update path (if that's still even needed) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecChangeBootDiskSize( + vmCtx context.VirtualMachineContextA2, + _ *VMCreateArgs) error { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil + } + + // TODO: How to we determine the DeviceKey for the DeviceChange entry? We probably have to + // crack the image/source, which is hard to do ATM. Punt on this for a placement consideration + // and we'll resize the boot (first) disk after VM create like before. + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecZipNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + if vmCtx.VM.Spec.Network.Disabled { + util.RemoveDevicesFromConfigSpec(createArgs.ConfigSpec, util.IsEthernetCard) + return nil + } + + resultsIdx := 0 + var unmatchedEthDevices []int + + for idx := range createArgs.ConfigSpec.DeviceChange { + spec := createArgs.ConfigSpec.DeviceChange[idx].GetVirtualDeviceConfigSpec() + if spec == nil || !util.IsEthernetCard(spec.Device) { + continue + } + + device := spec.Device + ethCard := device.(types.BaseVirtualEthernetCard).GetVirtualEthernetCard() + + if resultsIdx < len(createArgs.NetworkResults.Results) { + err := network.ApplyInterfaceResultToVirtualEthCard(vmCtx, ethCard, &createArgs.NetworkResults.Results[resultsIdx]) + if err != nil { + return err + } + resultsIdx++ + + } else { + // This ConfigSpec Ethernet device does not have a corresponding entry in the VM Spec, so we + // won't ever have a backing for it. Remove it from the ConfigSpec since that is the easiest + // thing to do, since extra NICs can cause later complications around GOSC and other customizations. + // The downside with this is that if a NIC is added to the VM Spec, it won't necessarily have this + // config but the default. Revisit this later if we don't like that behavior. + unmatchedEthDevices = append(unmatchedEthDevices, idx-len(unmatchedEthDevices)) + } + } + + if len(unmatchedEthDevices) > 0 { + deviceChange := createArgs.ConfigSpec.DeviceChange + for _, idx := range unmatchedEthDevices { + deviceChange = append(deviceChange[:idx], deviceChange[idx+1:]...) + } + createArgs.ConfigSpec.DeviceChange = deviceChange + } + + // Any remaining VM Spec network interfaces were not matched with a device in the ConfigSpec, so + // create a default virtual ethernet card for them. + for i := resultsIdx; i < len(createArgs.NetworkResults.Results); i++ { + ethCardDev, err := network.CreateDefaultEthCard(vmCtx, &createArgs.NetworkResults.Results[i]) + if err != nil { + return err + } + + // May not have the backing yet (NSX-T). We come back through here after placement once we + // know the backing. + if ethCardDev != nil { + createArgs.ConfigSpec.DeviceChange = append(createArgs.ConfigSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: ethCardDev, + }) + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateValidateArgs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + // Some of this would be better done in the validation webhook but have it here for now. + cfg := vcClient.Config() + + if cfg.StorageClassRequired { + // In WCP this is always required. + if vmCtx.VM.Spec.StorageClass == "" { + return fmt.Errorf("StorageClass is required but not specified") + } + + if createArgs.StorageProfileID == "" { + // GetVMStoragePoliciesIDs() would have returned an error if the policy didn't exist, but + // ensure the field is set. + return fmt.Errorf("no StorageProfile found for StorageClass %s", vmCtx.VM.Spec.StorageClass) + } + + } else if vmCtx.VM.Spec.StorageClass == "" { + // This is only set in gce2e. + if cfg.Datastore == "" { + return fmt.Errorf("no Datastore provided in configuration") + } + + datastore, err := vcClient.Finder().Datastore(vmCtx, cfg.Datastore) + if err != nil { + return fmt.Errorf("failed to find Datastore %s: %w", cfg.Datastore, err) + } + + createArgs.DatastoreMoID = datastore.Reference().Value + } + + return nil +} + +func (vs *vSphereVMProvider) vmUpdateGetArgs( + vmCtx context.VirtualMachineContextA2) (*vmUpdateArgs, error) { + + vmClass, err := GetVirtualMachineClass(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + resourcePolicy, err := GetVMSetResourcePolicy(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + data, vAppData, vAppExData, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + updateArgs := &vmUpdateArgs{} + updateArgs.VMClass = vmClass + updateArgs.ResourcePolicy = resourcePolicy + updateArgs.BootstrapData.Data = data + updateArgs.BootstrapData.VAppData = vAppData + updateArgs.BootstrapData.VAppExData = vAppExData + + if res := vmClass.Spec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + freq, err := vs.getOrComputeCPUMinFrequency(vmCtx) + if err != nil { + return nil, err + } + updateArgs.MinCPUFreq = freq + } + + var vmClassConfigSpec *types.VirtualMachineConfigSpec + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if cs := updateArgs.VMClass.Spec.ConfigSpec; cs != nil { + var err error + vmClassConfigSpec, err = GetVMClassConfigSpec(cs) + if err != nil { + return nil, err + } + } + } + + var vmImageStatus *vmopv1.VirtualMachineImageStatus + // Only get VM image when this is the VM first boot. + if isVMFirstBoot(vmCtx) { + var err error + _, _, vmImageStatus, err = GetVirtualMachineImageSpecAndStatus(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + // The only use of this is for the global JSON_EXTRA_CONFIG to set the image name. + // The global extra config should only be set during first boot. + // TODO: We can just finally kill this with the demise of old gce2e? + renderTemplateFn := func(name, text string) string { + t, err := template.New(name).Parse(text) + if err != nil { + return text + } + b := strings.Builder{} + if err := t.Execute(&b, vmImageStatus); err != nil { + return text + } + return b.String() + } + extraConfig := make(map[string]string, len(vs.globalExtraConfig)) + for k, v := range vs.globalExtraConfig { + extraConfig[k] = renderTemplateFn(k, v) + } + updateArgs.ExtraConfig = extraConfig + + // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot + // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled + // when the vmMetadata is nil or when the transport requested is not CloudInit. + // TODO: Is this still actually needed? + updateArgs.VirtualMachineImageV1Alpha1Compatible = + conditions.IsTrueFromConditions(vmImageStatus.Conditions, "VirtualMachineImageV1Alpha1Compatible" /*vmopv1.VirtualMachineImageV1Alpha1CompatibleCondition*/) + } + + updateArgs.ConfigSpec = virtualmachine.CreateConfigSpec( + vmCtx, + vmClassConfigSpec, + &updateArgs.VMClass.Spec, + vmImageStatus, + updateArgs.MinCPUFreq) + + return updateArgs, nil +} + +func isVMFirstBoot(vmCtx context.VirtualMachineContextA2) bool { + if _, ok := vmCtx.VM.Annotations[FirstBootDoneAnnotation]; ok { + return false + } + + return true +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go new file mode 100644 index 000000000..5b516a6cb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go @@ -0,0 +1,256 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + goctx "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +// vmE2ETests() tries to close the gap in the existing vmTests() have in the sense that we don't do e2e-like +// tests of the typical VM create/update workflow. This somewhat of a super-set of the vmTests() but those +// tests are already kind of unwieldy and in places, and until we switch over to v1a2, I don't +// to disturb that file so keeping things in sync easier. +// For now, these tests focus on a real - VDS or NSX-T - network env w/ cloud init, and we'll see how these +// need to evolve. +func vmE2ETests() { + + var ( + initObjects []client.Object + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + nsInfo builder.WorkloadNamespaceInfo + + vm *vmopv1.VirtualMachine + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + // Speed up tests until we Watch the network types. Sigh. + network.RetryTimeout = 1 * time.Millisecond + + testConfig = builder.VCSimTestConfig{ + WithV1A2: true, + WithContentLibrary: true, + WithVMClassAsConfigDaynDate: true, + } + + vm = builder.DummyBasicVirtualMachineA2("test-vm", "") + vmClass = builder.DummyVirtualMachineClassA2() + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + ctx.Context = goctx.WithValue(ctx.Context, context.MaxDeployThreadsContextKey, 1) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + nsInfo = ctx.CreateWorkloadNamespace() + + vmClass.Namespace = nsInfo.Namespace + Expect(ctx.Client.Create(ctx, vmClass)).To(Succeed()) + vmClass.Status.Ready = true + Expect(ctx.Client.Status().Update(ctx, vmClass)).To(Succeed()) + + cloudInitSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cloud-init-secret", + Namespace: nsInfo.Namespace, + }, + StringData: map[string]string{ + "user-value": "", + }, + } + Expect(ctx.Client.Create(ctx, cloudInitSecret)).To(Succeed()) + + vm.Namespace = nsInfo.Namespace + vm.Spec.ClassName = vmClass.Name + vm.Spec.ImageName = ctx.ContentLibraryImageName + vm.Spec.StorageClass = ctx.StorageClassName + vm.Spec.Network.Nameservers = []string{"1.1.1.1", "8.8.8.8"} + vm.Spec.Network.SearchDomains = []string{"vmware.local"} + vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{ + RawCloudConfig: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cloudInitSecret.Name, + }, + }, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + vmProvider = nil + nsInfo = builder.WorkloadNamespaceInfo{} + + vm = nil + vmClass = nil + }) + + Context("VDS", func() { + + const ( + networkName = "my-vds-network" + interfaceName = "eth0" + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvVDS + + vm.Spec.Network.Network = &common.PartialObjectRef{ + Name: networkName, + } + }) + + It("DoIt", func() { + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("simulate successful NetOP reconcile", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.NetworkName).To(Equal(networkName)) + + netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value + netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.110", + IPFamily: netopv1alpha1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + devList, err := vcVM.Device(ctx) + Expect(err).ToNot(HaveOccurred()) + + // For now just check the expected Nic backing. + By("Has expected NIC backing", func() { + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + + dev1 := l[0].GetVirtualDevice() + backingInfo, ok := dev1.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + }) + }) + + Context("NSX-T", func() { + + const ( + networkName = "my-nsxt-network" + interfaceName = "eth0" + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + + vm.Spec.Network.Network = &common.PartialObjectRef{ + Name: networkName, + } + }) + + It("DoIt", func() { + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("simulate successful NCP reconcile", func() { + netInterface := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.VirtualNetwork).To(Equal(networkName)) + + netInterface.Status.MacAddress = "01-23-45-67-89-AB-CD-EF" + netInterface.Status.ProviderStatus = &ncpv1alpha1.VirtualNetworkInterfaceProviderStatus{ + NsxLogicalSwitchID: builder.NsxTLogicalSwitchUUID, + } + netInterface.Status.IPAddresses = []ncpv1alpha1.VirtualNetworkInterfaceIP{ + { + IP: "192.168.1.110", + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + } + netInterface.Status.Conditions = []ncpv1alpha1.VirtualNetworkCondition{ + { + Type: "Ready", + Status: "True", + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + devList, err := vcVM.Device(ctx) + Expect(err).ToNot(HaveOccurred()) + + // For now just check the expected Nic backing. + By("Has expected NIC backing", func() { + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + + dev1 := l[0].GetVirtualDevice() + backingInfo, ok := dev1.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go new file mode 100644 index 000000000..fbf325f43 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go @@ -0,0 +1,1813 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "bytes" + goctx "context" + "encoding/json" + "fmt" + "math/rand" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vapi/cluster" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vmTests() { + + const ( + // Hardcoded vcsim CPU frequency. + vcsimCPUFreq = 2294 + + // Default network created for free by vcsim. + dvpgName = "DC0_DVPG0" + ) + + var ( + initObjects []client.Object + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + nsInfo builder.WorkloadNamespaceInfo + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + ctx.Context = goctx.WithValue(ctx.Context, context.MaxDeployThreadsContextKey, 1) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + vmProvider = nil + nsInfo = builder.WorkloadNamespaceInfo{} + }) + + Context("Create/Update/Delete VirtualMachine", func() { + var ( + vm *vmopv1.VirtualMachine + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + testConfig.WithContentLibrary = true + vmClass = builder.DummyVirtualMachineClassA2() + vm = builder.DummyBasicVirtualMachineA2("test-vm", "") + + // Reduce diff from old tests: by default don't create an NIC. + vm.Spec.Network.Disabled = true + }) + + AfterEach(func() { + vmClass = nil + vm = nil + }) + + JustBeforeEach(func() { + vmClass.Namespace = nsInfo.Namespace + Expect(ctx.Client.Create(ctx, vmClass)).To(Succeed()) + vmClass.Status.Ready = true + Expect(ctx.Client.Status().Update(ctx, vmClass)).To(Succeed()) + + clusterVMImage := &vmopv1.ClusterVirtualMachineImage{} + if testConfig.WithContentLibrary { + Expect(ctx.Client.Get(ctx, client.ObjectKey{Name: ctx.ContentLibraryImageName}, clusterVMImage)).To(Succeed()) + } else { + // BMV: VM creation without CL is broken - and has been for a long while - since we assume + // the VM Image will always point to a ContentLibrary item. + // Hack around that with this knob so we can continue to test the VM clone path. + vsphere.SkipVMImageCLProviderCheck = true + + // Use the default VM created by vcsim as the source. + clusterVMImage = builder.DummyClusterVirtualMachineImageA2("DC0_C0_RP0_VM0") + Expect(ctx.Client.Create(ctx, clusterVMImage)).To(Succeed()) + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) + Expect(ctx.Client.Status().Update(ctx, clusterVMImage)).To(Succeed()) + } + + vm.Namespace = nsInfo.Namespace + vm.Spec.ClassName = vmClass.Name + vm.Spec.ImageName = clusterVMImage.Name + vm.Spec.StorageClass = ctx.StorageClassName + }) + + AfterEach(func() { + vsphere.SkipVMImageCLProviderCheck = false + }) + + createOrUpdateAndGetVcVM := func( + ctx *builder.TestContextForVCSim, + vm *vmopv1.VirtualMachine) (*object.VirtualMachine, error) { + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + if err != nil { + return nil, err + } + + ExpectWithOffset(1, vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + ExpectWithOffset(1, vcVM).ToNot(BeNil()) + return vcVM, nil + } + + Context("VMClassAsConfigDaynDate FSS is enabled", func() { + + var ( + vcVM *object.VirtualMachine + configSpec *types.VirtualMachineConfigSpec + ethCard types.VirtualEthernetCard + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + testConfig.WithVMClassAsConfigDaynDate = true + + ethCard = types.VirtualEthernetCard{ + VirtualDevice: types.VirtualDevice{ + Key: 4000, + DeviceInfo: &types.Description{ + Label: "test-configspec-nic-label", + Summary: "VM Network", + }, + SlotInfo: &types.VirtualDevicePciBusSlotInfo{ + VirtualDeviceBusSlotInfo: types.VirtualDeviceBusSlotInfo{}, + PciSlotNumber: 32, + }, + ControllerKey: 100, + }, + AddressType: string(types.VirtualEthernetCardMacTypeGenerated), + MacAddress: "00:0c:29:93:d7:27", + ResourceAllocation: &types.VirtualEthernetCardResourceAllocation{ + Reservation: pointer.Int64(42), + }, + } + }) + + JustBeforeEach(func() { + if configSpec != nil { + var w bytes.Buffer + enc := types.NewJSONEncoder(&w) + Expect(enc.Encode(configSpec)).To(Succeed()) + + // Update the VM Class with the XML. + vmClass.Spec.ConfigSpec = w.Bytes() + Expect(ctx.Client.Update(ctx, vmClass)).To(Succeed()) + } + + vm.Spec.Network.Disabled = false + vm.Spec.Network.Network = &common.PartialObjectRef{Name: dvpgName} + + var err error + vcVM, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + vcVM = nil + configSpec = nil + }) + + Context("VM Class has no ConfigSpec", func() { + BeforeEach(func() { + configSpec = nil + }) + + It("creates VM", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + }) + + Context("ConfigSpec specifies hardware spec", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{ + Name: "config-spec-name-is-not-used", + NumCPUs: 7, + MemoryMB: 5102, + } + }) + + It("CPU and memory from ConfigSpec are ignored", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Summary.Config.Name).To(Equal(vm.Name)) + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.NumCpu).ToNot(BeEquivalentTo(configSpec.NumCPUs)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + Expect(o.Summary.Config.MemorySizeMB).ToNot(BeEquivalentTo(configSpec.MemoryMB)) + }) + }) + + Context("VM Class spec CPU reservation & limits are non-zero and ConfigSpec specifies CPU reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Cpu = resource.MustParse("2") + vmClass.Spec.Policies.Resources.Limits.Cpu = resource.MustParse("3") + + // Specify a CPU reservation via ConfigSpec. This value should not be honored. + configSpec = &types.VirtualMachineConfigSpec{ + CpuAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(6), + }, + } + }) + + It("VM gets CPU reservation from VM Class spec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + resources := &vmClass.Spec.Policies.Resources + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.CPUQuantityToMhz(resources.Requests.Cpu, vcsimCPUFreq))) + Expect(*reservation).ToNot(Equal(*configSpec.CpuAllocation.Reservation)) + + limit := o.Config.CpuAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.CPUQuantityToMhz(resources.Limits.Cpu, vcsimCPUFreq))) + }) + }) + + Context("VM Class spec CPU reservation is zero and ConfigSpec specifies CPU reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Cpu = resource.MustParse("0") + vmClass.Spec.Policies.Resources.Limits.Cpu = resource.MustParse("0") + + // Specify a CPU reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + CpuAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(6), + }, + } + }) + + It("VM gets CPU reservation from ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).ToNot(BeZero()) + Expect(*reservation).To(Equal(*configSpec.CpuAllocation.Reservation)) + }) + }) + + Context("VM Class spec Memory reservation & limits are non-zero and ConfigSpec specifies memory reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Memory = resource.MustParse("4Mi") + vmClass.Spec.Policies.Resources.Limits.Memory = resource.MustParse("4Mi") + + // Specify a Memory reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + MemoryAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(5120), + }, + } + }) + + It("VM gets memory reservation from VM Class spec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + resources := &vmClass.Spec.Policies.Resources + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.MemoryQuantityToMb(resources.Requests.Memory))) + Expect(*reservation).ToNot(Equal(*configSpec.MemoryAllocation.Reservation)) + + limit := o.Config.MemoryAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.MemoryQuantityToMb(resources.Limits.Memory))) + }) + }) + + Context("VM Class spec Memory reservations are zero and ConfigSpec specifies memory reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Memory = resource.MustParse("0Mi") + vmClass.Spec.Policies.Resources.Limits.Memory = resource.MustParse("0Mi") + + // Specify a Memory reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + MemoryAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(5120), + }, + } + }) + + It("VM gets memory reservation from ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).ToNot(BeZero()) + Expect(*reservation).To(Equal(*configSpec.MemoryAllocation.Reservation)) + }) + }) + + Context("VM Class ConfigSpec specifies a network interface", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + // Create the ConfigSpec with an ethernet card. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualE1000{ + VirtualEthernetCard: ethCard, + }, + }, + }, + } + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with the NIC specified in ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + + ethDevice, ok := l[0].(*types.VirtualE1000) + // ethDevice, ok := l[0+1].(*types.VirtualE1000) + Expect(ok).To(BeTrue()) + Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) + Expect(ethDevice.MacAddress).To(Equal(ethCard.MacAddress)) + + Expect(dev.DeviceInfo).To(Equal(ethCard.VirtualDevice.DeviceInfo)) + Expect(dev.DeviceGroupInfo).To(Equal(ethCard.VirtualDevice.DeviceGroupInfo)) + Expect(dev.SlotInfo).To(Equal(ethCard.VirtualDevice.SlotInfo)) + Expect(dev.ControllerKey).To(Equal(ethCard.VirtualDevice.ControllerKey)) + Expect(ethDevice.ResourceAllocation).ToNot(BeNil()) + Expect(ethDevice.ResourceAllocation.Reservation).ToNot(BeNil()) + Expect(*ethDevice.ResourceAllocation.Reservation).To(Equal(*ethCard.ResourceAllocation.Reservation)) + }) + }) + + Context("ConfigSpec does not specify any network interfaces", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + configSpec = &types.VirtualMachineConfigSpec{} + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with the default NIC settings from provider", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + }) + }) + + Context("VM Class Spec and ConfigSpec both contain GPU and DirectPath devices", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.Devices = vmopv1.VirtualDevices{ + VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "profile-from-class", + }, + }, + DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 50, + DeviceID: 51, + CustomLabel: "label-from-class", + }, + }, + } + + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-config-spec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-config-spec", + }, + }, + }, + }, + }, + } + }) + + It("GPU and DirectPath devices from VM Class Spec.Devices are ignored", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("profile-from-config-spec")) + + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(52))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(53))) + Expect(pciBacking2.CustomLabel).To(Equal("label-from-config-spec")) + }) + }) + + Context("VM Class Config specifies an ethCard, a GPU and a DDPIO device", func() { + + BeforeEach(func() { + // Create the ConfigSpec with an ethernet card, a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualE1000{ + VirtualEthernetCard: ethCard, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "SampleLabel2", + }, + }, + }, + }, + }, + } + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with a NIC, GPU and DDPIO device specified in ConfigSpec", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + + ethDevice, ok := l[0].(*types.VirtualE1000) + // ethDevice, ok := l[0+1].(*types.VirtualE1000) + Expect(ok).To(BeTrue()) + Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) + Expect(dev.DeviceInfo).To(Equal(ethCard.VirtualDevice.DeviceInfo)) + Expect(dev.DeviceGroupInfo).To(Equal(ethCard.VirtualDevice.DeviceGroupInfo)) + Expect(dev.SlotInfo).To(Equal(ethCard.VirtualDevice.SlotInfo)) + Expect(dev.ControllerKey).To(Equal(ethCard.VirtualDevice.ControllerKey)) + Expect(ethDevice.MacAddress).To(Equal(ethCard.MacAddress)) + Expect(ethDevice.ResourceAllocation).ToNot(BeNil()) + Expect(ethDevice.ResourceAllocation.Reservation).ToNot(BeNil()) + Expect(*ethDevice.ResourceAllocation.Reservation).To(Equal(*ethCard.ResourceAllocation.Reservation)) + + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("SampleProfile2")) + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(52))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(53))) + Expect(pciBacking2.CustomLabel).To(Equal("SampleLabel2")) + + // CPU and memory should be from vm class + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + }) + + Context("VM Class Config specifies disks, disk controllers, other miscellaneous devices", func() { + BeforeEach(func() { + // Create the ConfigSpec with disks, disk controller and some misc devices: pointing device, + // video card, etc. This works fine with vcsim and helps with testing adding misc devices. + // The simulator can still reconfigure the VM with default device types like pointing devices, + // keyboard, video card, etc. But VC has some restrictions with reconfiguring a VM with new + // default device types via ConfigSpec and are usually ignored. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPointingDevice{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPointingDeviceDeviceBackingInfo{ + HostPointingDevice: "autodetect", + }, + Key: 700, + ControllerKey: 300, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPS2Controller{ + VirtualController: types.VirtualController{ + Device: []int32{700}, + VirtualDevice: types.VirtualDevice{ + Key: 300, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualMachineVideoCard{ + UseAutoDetect: pointer.Bool(false), + NumDisplays: 1, + VirtualDevice: types.VirtualDevice{ + Key: 500, + ControllerKey: 100, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIController{ + VirtualController: types.VirtualController{ + Device: []int32{500}, + VirtualDevice: types.VirtualDevice{ + Key: 100, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSCSIController{ + VirtualController: types.VirtualController{ + Device: []int32{-42}, + }, + }, + }, + }, + } + }) + + // FIXME: vcsim behavior needs to be closer to real VC here so there aren't dupes + It("Reconfigures the VM with all misc devices in ConfigSpec except disk and disk controllers", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + + // VM already has a default pointing device and the spec adds one more + // info about the default device is unknown to assert on + pointingDev := devList.SelectByType(&types.VirtualPointingDevice{}) + Expect(pointingDev).To(HaveLen(2)) + dev := pointingDev[0].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualPointingDeviceDeviceBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backing.HostPointingDevice).To(Equal("autodetect")) + Expect(dev.Key).To(Equal(int32(700))) + Expect(dev.ControllerKey).To(Equal(int32(300))) + + ps2Controllers := devList.SelectByType(&types.VirtualPS2Controller{}) + Expect(ps2Controllers).To(HaveLen(1)) + dev = ps2Controllers[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(300))) + + pciControllers := devList.SelectByType(&types.VirtualPCIController{}) + Expect(pciControllers).To(HaveLen(1)) + dev = pciControllers[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(100))) + + // VM already has a default video card and the spec adds one more + // info about the default device is unknown to assert on + video := devList.SelectByType(&types.VirtualMachineVideoCard{}) + Expect(video).To(HaveLen(2)) + dev = video[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(500))) + Expect(dev.ControllerKey).To(Equal(int32(100))) + + // Disk and disk controllers from config spec should not get added, since we + // filter them out in our ConfigSpec + diskControllers := devList.SelectByType(&types.VirtualSCSIController{}) + Expect(diskControllers).To(BeEmpty()) + + // Only preexisting disk should be present on VM -- len: 1 + disks := devList.SelectByType(&types.VirtualDisk{}) + Expect(disks).To(HaveLen(1)) + dev = disks[0].GetVirtualDevice() + Expect(dev.Key).ToNot(Equal(int32(-42))) + }) + }) + + Context("VM Class Config does not specify a hardware version", func() { + + Context("VM Class has vGPU and/or DDPIO devices", func() { + BeforeEach(func() { + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PCI devices", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPCIPassthruDevices))) + }) + }) + + Context("VM Class has vGPU and/or DDPIO devices and VM spec has a PVC", func() { + BeforeEach(func() { + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + } + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "dummy-vol", + Attached: true, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PCI devices", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPCIPassthruDevices))) + }) + }) + + Context("VM spec has a PVC", func() { + BeforeEach(func() { + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + } + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "dummy-vol", + Attached: true, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PVCs", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPVC))) + }) + }) + }) + + Context("VMClassAsConfig FSS is Enabled", func() { + + BeforeEach(func() { + testConfig.WithVMClassAsConfig = true + }) + + When("configSpec has disk and disk controllers", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSATAController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 101, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSCSIController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 103, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualNVMEController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 104, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + }, + }, + } + }) + + It("creates a VM with disk controllers", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + satacont := devList.SelectByType(&types.VirtualSATAController{}) + Expect(satacont).To(HaveLen(1)) + dev := satacont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(101))) + + scsicont := devList.SelectByType(&types.VirtualSCSIController{}) + Expect(scsicont).To(HaveLen(1)) + dev = scsicont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(103))) + + nvmecont := devList.SelectByType(&types.VirtualNVMEController{}) + Expect(nvmecont).To(HaveLen(1)) + dev = nvmecont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(104))) + + // only preexisting disk should be present on VM -- len: 1 + disks := devList.SelectByType(&types.VirtualDisk{}) + Expect(disks).To(HaveLen(1)) + dev1 := disks[0].GetVirtualDevice() + Expect(dev1.Key).ToNot(Equal(int32(-42))) + }) + }) + }) + }) + + Context("CreateOrUpdate VM", func() { + + It("Basic VM", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected Status values", func() { + Expect(vm.Status.PowerState).To(Equal(vm.Spec.PowerState)) + Expect(vm.Status.Host).ToNot(BeEmpty()) + Expect(vm.Status.InstanceUUID).To(And(Not(BeEmpty()), Equal(o.Config.InstanceUuid))) + Expect(vm.Status.BiosUUID).To(And(Not(BeEmpty()), Equal(o.Config.Uuid))) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionClassReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionStorageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix(fmt.Sprintf("/%s/%s", nsInfo.Namespace, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + + By("has expected power state", func() { + Expect(o.Summary.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + }) + + vmClassRes := &vmClass.Spec.Policies.Resources + + By("has expected CpuAllocation", func() { + Expect(o.Config.CpuAllocation).ToNot(BeNil()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.CPUQuantityToMhz(vmClassRes.Requests.Cpu, vcsimCPUFreq))) + limit := o.Config.CpuAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.CPUQuantityToMhz(vmClassRes.Limits.Cpu, vcsimCPUFreq))) + }) + + By("has expected MemoryAllocation", func() { + Expect(o.Config.MemoryAllocation).ToNot(BeNil()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.MemoryQuantityToMb(vmClassRes.Requests.Memory))) + limit := o.Config.MemoryAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.MemoryQuantityToMb(vmClassRes.Limits.Memory))) + }) + + By("has expected hardware config", func() { + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + + // TODO: More assertions! + }) + + Context("VM Class with PCI passthrough devices", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.Devices = vmopv1.VirtualDevices{ + VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "profile-from-class-without-class-as-config-fss", + }, + }, + DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 59, + DeviceID: 60, + CustomLabel: "label-from-class-without-class-as-config-fss", + }, + }, + } + }) + + It("VM should have expected PCI devices from VM Class", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("profile-from-class-without-class-as-config-fss")) + + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(59))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(60))) + Expect(pciBacking2.CustomLabel).To(Equal("label-from-class-without-class-as-config-fss")) + }) + }) + + Context("Without Storage Class", func() { + BeforeEach(func() { + testConfig.WithoutStorageClass = true + }) + + It("Creates VM", func() { + Expect(vm.Spec.StorageClass).To(BeEmpty()) + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected datastore", func() { + datastore, err := ctx.Finder.DefaultDatastore(ctx) + Expect(err).ToNot(HaveOccurred()) + + Expect(o.Datastore).To(HaveLen(1)) + Expect(o.Datastore[0]).To(Equal(datastore.Reference())) + }) + }) + }) + + Context("Without Content Library", func() { + BeforeEach(func() { + testConfig.WithContentLibrary = false + }) + + // TODO: Dedupe this with "Basic VM" above + It("Clones VM", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected Status values", func() { + Expect(vm.Status.PowerState).To(Equal(vm.Spec.PowerState)) + Expect(vm.Status.Host).ToNot(BeEmpty()) + Expect(vm.Status.InstanceUUID).To(And(Not(BeEmpty()), Equal(o.Config.InstanceUuid))) + Expect(vm.Status.BiosUUID).To(And(Not(BeEmpty()), Equal(o.Config.Uuid))) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionClassReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionStorageReady)).To(BeTrue()) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix(fmt.Sprintf("/%s/%s", nsInfo.Namespace, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + + By("has expected power state", func() { + Expect(o.Summary.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + }) + + By("has expected hardware config", func() { + // TODO: Fix vcsim behavior: NumCPU is correct "2" in the CloneSpec.Config but ends up + // with 1 CPU from source VM. Ditto for MemorySize. These assertions are only working + // because the state is on so we reconfigure the VM after it is created. + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + + // TODO: More assertions! + }) + }) + + // BMV: I don't think this is actually supported. + XIt("Create VM from VMTX in ContentLibrary", func() { + imageName := "test-vm-vmtx" + + ctx.ContentLibraryItemTemplate("DC0_C0_RP0_VM0", imageName) + vm.Spec.ImageName = imageName + + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("When fault domains is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + It("creates VM in placement selected zone", func() { + Expect(vm.Labels).ToNot(HaveKey(topology.KubernetesTopologyZoneLabelKey)) + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + azName, ok := vm.Labels[topology.KubernetesTopologyZoneLabelKey] + Expect(ok).To(BeTrue()) + Expect(azName).To(BeElementOf(ctx.ZoneNames)) + + By("VM is created in the zone's ResourcePool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, azName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + }) + + It("creates VM in assigned zone", func() { + azName := ctx.ZoneNames[rand.Intn(len(ctx.ZoneNames))] //nolint:gosec + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = azName + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + By("VM is created in the zone's ResourcePool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, azName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + }) + }) + + Context("When Instance Storage FSS is enabled", func() { + BeforeEach(func() { + testConfig.WithInstanceStorage = true + }) + + expectInstanceStorageVolumes := func( + vm *vmopv1.VirtualMachine, + isStorage vmopv1.InstanceStorage) { + + ExpectWithOffset(1, isStorage.Volumes).ToNot(BeEmpty()) + isVolumes := instancestorage.FilterVolumes(vm) + ExpectWithOffset(1, isVolumes).To(HaveLen(len(isStorage.Volumes))) + + for _, isVol := range isStorage.Volumes { + found := false + + for idx, vol := range isVolumes { + claim := vol.PersistentVolumeClaim.InstanceVolumeClaim + if claim.StorageClass == isStorage.StorageClass && claim.Size == isVol.Size { + isVolumes = append(isVolumes[:idx], isVolumes[idx+1:]...) + found = true + break + } + } + + ExpectWithOffset(1, found).To(BeTrue(), "failed to find instance storage volume for %v", isVol) + } + } + + It("creates VM without instance storage", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + It("create VM with instance storage", func() { + Expect(vm.Spec.Volumes).To(BeEmpty()) + + vmClass.Spec.Hardware.InstanceStorage = vmopv1.InstanceStorage{ + StorageClass: vm.Spec.StorageClass, + Volumes: []vmopv1.InstanceStorageVolume{ + { + Size: resource.MustParse("256Gi"), + }, + { + Size: resource.MustParse("512Gi"), + }, + }, + } + Expect(ctx.Client.Update(ctx, vmClass)).To(Succeed()) + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("instance storage PVCs are not bound yet")) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeFalse()) + + By("Instance storage volumes should be added to VM", func() { + Expect(instancestorage.IsPresent(vm)).To(BeTrue()) + expectInstanceStorageVolumes(vm, vmClass.Spec.Hardware.InstanceStorage) + }) + + By("Placement should have been done", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionPlacementReady)).To(BeTrue()) + Expect(vm.Annotations).To(HaveKey(constants.InstanceStorageSelectedNodeAnnotationKey)) + Expect(vm.Annotations).To(HaveKey(constants.InstanceStorageSelectedNodeMOIDAnnotationKey)) + }) + + isVol0 := vm.Spec.Volumes[0] + Expect(isVol0.PersistentVolumeClaim.InstanceVolumeClaim).ToNot(BeNil()) + + By("simulate volume controller workflow", func() { + // Simulate what would be set by volume controller. + vm.Annotations[constants.InstanceStoragePVCsBoundAnnotationKey] = "" + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("status update pending for persistent volume: %s on VM", isVol0.Name))) + + // Simulate what would be set by the volume controller. + for _, vol := range vm.Spec.Volumes { + vm.Status.Volumes = append(vm.Status.Volumes, vmopv1.VirtualMachineVolumeStatus{ + Name: vol.Name, + Attached: true, + }) + } + }) + + By("VM is now created", func() { + _, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + }) + }) + + It("Powers VM off", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOff)) + }) + + It("returns error when StorageClass is required but none specified", func() { + vm.Spec.StorageClass = "" + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("StorageClass is required but not specified")) + }) + + It("Can be called multiple times", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + modified := o.Config.Modified + + _, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + // Try to assert nothing changed. + Expect(o.Config.Modified).To(Equal(modified)) + }) + + Context("VM Metadata", func() { + + Context("ExtraConfig Transport", func() { + var ec map[string]interface{} + + JustBeforeEach(func() { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "md-configmap-", + Namespace: vm.Namespace, + }, + Data: map[string]string{ + "foo.bar": "should-be-ignored", + "guestinfo.Foo": "foo", + }, + } + Expect(ctx.Client.Create(ctx, configMap)).To(Succeed()) + + /* + vm.Spec.VmMetadata = &vmopv1.VirtualMachineMetadata{ + ConfigMapName: configMap.Name, + Transport: vmopv1.VirtualMachineMetadataExtraConfigTransport, + } + */ + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + ec = map[string]interface{}{} + for _, option := range o.Config.ExtraConfig { + if val := option.GetOptionValue(); val != nil { + ec[val.Key] = val.Value.(string) + } + } + }) + + AfterEach(func() { + ec = nil + }) + + // TODO: As is we can't really honor "guestinfo.*" prefix + XIt("Metadata data is included in ExtraConfig", func() { + Expect(ec).ToNot(HaveKey("foo.bar")) + Expect(ec).To(HaveKeyWithValue("guestinfo.Foo", "foo")) + + By("Should include default keys and values", func() { + Expect(ec).To(HaveKeyWithValue("disk.enableUUID", "TRUE")) + Expect(ec).To(HaveKeyWithValue("vmware.tools.gosc.ignoretoolscheck", "TRUE")) + }) + }) + + Context("JSON_EXTRA_CONFIG is specified", func() { + BeforeEach(func() { + b, err := json.Marshal( + struct { + Foo string + Bar string + }{ + Foo: "f00", + Bar: "42", + }, + ) + Expect(err).ToNot(HaveOccurred()) + testConfig.WithJSONExtraConfig = string(b) + }) + + It("Global config is included in ExtraConfig", func() { + Expect(ec).To(HaveKeyWithValue("Foo", "f00")) + Expect(ec).To(HaveKeyWithValue("Bar", "42")) + }) + }) + }) + }) + + Context("Network", func() { + + It("Should not have a nic", func() { + Expect(vm.Spec.Network.Disabled).To(BeTrue()) + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(BeEmpty()) + }) + + Context("Multiple NICs are specified", func() { + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + vm.Spec.Network.Disabled = false + vm.Spec.Network.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: "VM Network"}, + }, + { + Name: "eth1", + Network: common.PartialObjectRef{Name: dvpgName}, + }, + } + }) + + It("Has expected devices", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(2)) + + dev1 := l[0].GetVirtualDevice() + backing1, ok := dev1.Backing.(*types.VirtualEthernetCardNetworkBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backing1.DeviceName).To(Equal("VM Network")) + + dev2 := l[1].GetVirtualDevice() + backing2, ok := dev2.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing2.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + }) + }) + }) + + Context("Disks", func() { + + Context("VM has thin provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThin + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.ThinProvisioned).To(PointTo(BeTrue())) + }) + }) + + XContext("VM has thick provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThick + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + /* vcsim CL deploy has "thick" but that isn't reflected for this disk. */ + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.ThinProvisioned).To(PointTo(BeFalse())) + }) + }) + + XContext("VM has eager zero provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThickEagerZero + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + /* vcsim CL deploy has "eagerZeroedThick" but that isn't reflected for this disk. */ + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.EagerlyScrub).To(PointTo(BeTrue())) + }) + }) + + Context("Should resize root disk", func() { + newSize := resource.MustParse("4242Gi") + + It("Succeeds", func() { + vm.Spec.Advanced.BootDiskCapacity = newSize + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + disk, _ := getVMHomeDisk(ctx, vcVM, o) + Expect(disk.CapacityInBytes).To(BeEquivalentTo(newSize.Value())) + }) + }) + }) + + Context("CNS Volumes", func() { + cnsVolumeName := "cns-volume-1" + + It("CSI Volumes workflow", func() { + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn + By("Add CNS volume to VM", func() { + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: cnsVolumeName, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-volume-1", + }, + }, + }, + }, + } + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("status update pending for persistent volume: %s on VM", cnsVolumeName))) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + }) + + By("CNS volume is not attached", func() { + errMsg := "blah blah blah not attached" + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: cnsVolumeName, + Attached: false, + Error: errMsg, + }, + } + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("persistent volume: %s not attached to VM", cnsVolumeName))) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + }) + + By("CNS volume is attached", func() { + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: cnsVolumeName, + Attached: true, + }, + } + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + }) + }) + }) + + Context("When fault domains is enabled", func() { + const zoneName = "az-1" + + BeforeEach(func() { + testConfig.WithFaultDomains = true + // Explicitly place the VM into one of the zones that the test context will create. + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("Reverse lookups existing VM into correct zone", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + Expect(vm.Status.Zone).To(Equal(zoneName)) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + Expect(vm.Status.Zone).To(Equal(zoneName)) + }) + }) + }) + + Context("VM SetResourcePolicy", func() { + var resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + + JustBeforeEach(func() { + resourcePolicyName := "test-policy" + resourcePolicy = getVirtualMachineSetResourcePolicy(resourcePolicyName, nsInfo.Namespace) + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(ctx.Client.Create(ctx, resourcePolicy)).To(Succeed()) + + vm.Annotations["vsphere-cluster-module-group"] = resourcePolicy.Spec.ClusterModuleGroups[0] + vm.Spec.Reserved.ResourcePolicyName = resourcePolicy.Name + }) + + AfterEach(func() { + resourcePolicy = nil + }) + + It("VM is created in child Folder and ResourcePool", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix( + fmt.Sprintf("/%s/%s/%s", nsInfo.Namespace, resourcePolicy.Spec.Folder, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + childRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", resourcePolicy.Spec.ResourcePool.Name) + Expect(childRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(childRP.Reference().Value)) + }) + }) + + It("Cluster Modules", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + members, err := cluster.NewManager(ctx.RestClient).ListModuleMembers(ctx, resourcePolicy.Status.ClusterModules[0].ModuleUuid) + Expect(err).ToNot(HaveOccurred()) + Expect(members).To(ContainElements(vcVM.Reference())) + }) + + It("Returns error with non-existence cluster module", func() { + vm.Annotations["vsphere-cluster-module-group"] = "bogusClusterMod" + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("ClusterModule bogusClusterMod not found")) + }) + }) + + Context("Delete VM", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + Context("when the VM is off", func() { + BeforeEach(func() { + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + }) + + It("deletes the VM", func() { + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + }) + + It("when the VM is on", func() { + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + // This checks that we power off the VM prior to deletion. + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + + It("returns success when VM does not exist", func() { + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + }) + + Context("When fault domains is enabled", func() { + const zoneName = "az-1" + + BeforeEach(func() { + testConfig.WithFaultDomains = true + // Explicitly place the VM into one of the zones that the test context will create. + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("returns NotFound when VM does not exist", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("Deletes existing VM when zone info is missing", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + }) + }) + + Context("Guest Heartbeat", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return guest heartbeat", func() { + heartbeat, err := vmProvider.GetVirtualMachineGuestHeartbeat(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + // Just testing for property query: field not set in vcsim. + Expect(heartbeat).To(BeEmpty()) + }) + }) + + Context("Web console ticket", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return ticket", func() { + // vcsim doesn't implement this yet so expect an error. + _, err := vmProvider.GetVirtualMachineWebMKSTicket(ctx, vm, "foo") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not implement: AcquireTicket")) + }) + }) + + Context("VM hardware version", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return version", func() { + version, err := vmProvider.GetVirtualMachineHardwareVersion(ctx, vm) + Expect(err).NotTo(HaveOccurred()) + Expect(version).To(Equal(int32(9))) + }) + }) + }) +} + +// getVMHomeDisk gets the VM's "home" disk. It makes some assumptions about the backing and disk name. +func getVMHomeDisk( + ctx *builder.TestContextForVCSim, + vcVM *object.VirtualMachine, + o mo.VirtualMachine) (*types.VirtualDisk, *types.VirtualDiskFlatVer2BackingInfo) { + + ExpectWithOffset(1, vcVM.Name()).ToNot(BeEmpty()) + ExpectWithOffset(1, o.Datastore).ToNot(BeEmpty()) + var dso mo.Datastore + ExpectWithOffset(1, vcVM.Properties(ctx, o.Datastore[0], nil, &dso)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByBackingInfo(&types.VirtualDiskFlatVer2BackingInfo{ + VirtualDeviceFileBackingInfo: types.VirtualDeviceFileBackingInfo{ + FileName: fmt.Sprintf("[%s] %s/disk-0.vmdk", dso.Name, vcVM.Name()), + }, + }) + ExpectWithOffset(1, l).To(HaveLen(1)) + + disk := l[0].(*types.VirtualDisk) + backing := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo) + + return disk, backing +} + +//nolint:unparam +func getDVPG( + ctx *builder.TestContextForVCSim, + path string) (object.NetworkReference, *object.DistributedVirtualPortgroup) { + + network, err := ctx.Finder.Network(ctx, path) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + dvpg, ok := network.(*object.DistributedVirtualPortgroup) + ExpectWithOffset(1, ok).To(BeTrue()) + + return network, dvpg +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go new file mode 100644 index 000000000..5506214e8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go @@ -0,0 +1,335 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + "encoding/json" + "fmt" + + "github.com/google/uuid" + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" +) + +// TODO: This mostly just a placeholder until we spend time on something better. Individual types +// don't make much sense since we don't lump everything under a single prereq condition anymore. +func errToConditionReasonAndMessage(err error) (string, string) { + switch { + case apierrors.IsNotFound(err): + return "NotFound", err.Error() + case apierrors.IsForbidden(err): + return "Forbidden", err.Error() + case apierrors.IsInvalid(err): + return "Invalid", err.Error() + case apierrors.IsInternalError(err): + return "InternalError", err.Error() + default: + return "GetError", err.Error() + } +} + +func GetVirtualMachineClass( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (*vmopv1.VirtualMachineClass, error) { + + key := ctrlclient.ObjectKey{Name: vmCtx.VM.Spec.ClassName, Namespace: vmCtx.VM.Namespace} + vmClass := &vmopv1.VirtualMachineClass{} + if err := k8sClient.Get(vmCtx, key, vmClass); err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady, reason, msg) + return nil, err + } + + if !vmClass.Status.Ready { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady, + "NotReady", "VirtualMachineClass is not marked as Ready") + return nil, fmt.Errorf("VirtualMachineClass is not Ready") + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady) + + return vmClass, nil +} + +func GetVirtualMachineImageSpecAndStatus( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (ctrlclient.Object, *vmopv1.VirtualMachineImageSpec, *vmopv1.VirtualMachineImageStatus, error) { + + var obj ctrlclient.Object + var spec *vmopv1.VirtualMachineImageSpec + var status *vmopv1.VirtualMachineImageStatus + + key := ctrlclient.ObjectKey{Name: vmCtx.VM.Spec.ImageName, Namespace: vmCtx.VM.Namespace} + vmImage := &vmopv1.VirtualMachineImage{} + if err := k8sClient.Get(vmCtx, key, vmImage); err != nil { + clusterVMImage := &vmopv1.ClusterVirtualMachineImage{} + + if apierrors.IsNotFound(err) { + key.Namespace = "" + err = k8sClient.Get(vmCtx, key, clusterVMImage) + } + + if err != nil { + // Don't use the k8s error as-is as we don't know to prefer the NS or cluster scoped error message. + // This is the same error/message that the prior code used. + reason, msg := "NotFound", fmt.Sprintf("Failed to get the VM's image: %s", key.Name) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, reason, msg) + return nil, nil, nil, fmt.Errorf("%s: %w", msg, err) + } + + obj, spec, status = clusterVMImage, &clusterVMImage.Spec, &clusterVMImage.Status + } else { + obj, spec, status = vmImage, &vmImage.Spec, &vmImage.Status + } + + // TODO: Fix the image conditions so it just has a single Ready instead of bleeding the CL stuff. + if !conditions.IsTrueFromConditions(status.Conditions, vmopv1.VirtualMachineImageSyncedCondition) { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, + "NotReady", "VirtualMachineImage is not ready") + return nil, nil, nil, fmt.Errorf("VirtualMachineImage is not ready") + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady) + + return obj, spec, status, nil +} + +func getSecretData( + vmCtx context.VirtualMachineContextA2, + name string, + cmFallback bool, + k8sClient ctrlclient.Client) (map[string]string, error) { + + var data map[string]string + + key := ctrlclient.ObjectKey{Name: name, Namespace: vmCtx.VM.Namespace} + secret := &corev1.Secret{} + if err := k8sClient.Get(vmCtx, key, secret); err != nil { + configMap := &corev1.ConfigMap{} + + // For backwards compat if we cannot find the Secret, fallback to a ConfigMap. In v1a1, either a + // Secret and ConfigMap was supported for metadata (bootstrap) as separate fields, but v1a2 only + // supports Secrets. + if cmFallback && apierrors.IsNotFound(err) { + err = k8sClient.Get(vmCtx, key, configMap) + } + + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, err + } + + data = configMap.Data + } else { + data = make(map[string]string, len(secret.Data)) + + for k, v := range secret.Data { + data[k] = string(v) + } + } + + return data, nil +} + +func GetVirtualMachineBootstrap( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (map[string]string, map[string]string, map[string]map[string]string, error) { + + bootstrapSpec := &vmCtx.VM.Spec.Bootstrap + var secretName string + var data, vAppData map[string]string + var vAppExData map[string]map[string]string + + if cloudInit := bootstrapSpec.CloudInit; cloudInit != nil { + secretName = cloudInit.RawCloudConfig.Name + } else if sysPrep := bootstrapSpec.Sysprep; sysPrep != nil { + secretName = sysPrep.RawSysprep.Name + } + + if secretName != "" { + var err error + + data, err = getSecretData(vmCtx, secretName, true, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + } + + // vApp bootstrap can be used alongside LinuxPrep/Sysprep. + if vApp := bootstrapSpec.VAppConfig; vApp != nil { + + if vApp.RawProperties != "" { + var err error + + vAppData, err = getSecretData(vmCtx, vApp.RawProperties, true, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + + } else { + for _, p := range vApp.Properties { + from := p.Value.From + if from == nil { + continue + } + + if _, ok := vAppExData[from.Name]; !ok { + // Do the easy thing here and carry along each Secret's entire data. We could instead + // shoehorn this in the vAppData with a concat key using an invalid k8s name delimiter. + // TODO: Check that key exists, and/or deal with from.Optional. Too many options. + fromData, err := getSecretData(vmCtx, from.Name, false, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + + if vAppExData == nil { + vAppExData = make(map[string]map[string]string) + } + vAppExData[from.Name] = fromData + } + } + } + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady) + + return data, vAppData, vAppExData, nil +} + +func GetVMSetResourcePolicy( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (*vmopv1.VirtualMachineSetResourcePolicy, error) { + + rpName := vmCtx.VM.Spec.Reserved.ResourcePolicyName + if rpName == "" { + conditions.Delete(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady) + return nil, nil + } + + key := ctrlclient.ObjectKey{Name: rpName, Namespace: vmCtx.VM.Namespace} + resourcePolicy := &vmopv1.VirtualMachineSetResourcePolicy{} + if err := k8sClient.Get(vmCtx, key, resourcePolicy); err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady, reason, msg) + return nil, err + } + + // The VirtualMachineSetResourcePolicy doesn't have a Ready condition or field but don't + // allow a VM to use a policy that's being deleted. + if !resourcePolicy.DeletionTimestamp.IsZero() { + err := fmt.Errorf("VirtualMachineSetResourcePolicy is being deleted") + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady, + "NotReady", err.Error()) + return nil, err + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady) + + return resourcePolicy, nil +} + +// AddInstanceStorageVolumes checks if VM class is configured with instance storage volumes and appends the +// volumes to the VM's Spec if not already done. Return true if the VM had or now has instance storage volumes. +func AddInstanceStorageVolumes( + vmCtx context.VirtualMachineContextA2, + vmClass *vmopv1.VirtualMachineClass) bool { + + if instancestorage.IsPresent(vmCtx.VM) { + // Instance storage disks are copied from the class to the VM only once, regardless + // if the class changes. + return true + } + + is := vmClass.Spec.Hardware.InstanceStorage + if len(is.Volumes) == 0 { + return false + } + + volumes := make([]vmopv1.VirtualMachineVolume, 0, len(is.Volumes)) + + for _, isv := range is.Volumes { + name := constants.InstanceStoragePVCNamePrefix + uuid.NewString() + + vmv := vmopv1.VirtualMachineVolume{ + Name: name, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + ReadOnly: false, + }, + InstanceVolumeClaim: &vmopv1.InstanceVolumeClaimVolumeSource{ + StorageClass: is.StorageClass, + Size: isv.Size, + }, + }, + }, + } + volumes = append(volumes, vmv) + } + + vmCtx.VM.Spec.Volumes = append(vmCtx.VM.Spec.Volumes, volumes...) + return true +} + +func GetVMClassConfigSpec(raw json.RawMessage) (*types.VirtualMachineConfigSpec, error) { + classConfigSpec, err := util.UnmarshalConfigSpecFromJSON(raw) + if err != nil { + return nil, err + } + util.SanitizeVMClassConfigSpec(classConfigSpec) + + return classConfigSpec, nil +} + +// HasPVC returns true if the VirtualMachine spec has a Persistent Volume claim. +func HasPVC(vmSpec vmopv1.VirtualMachineSpec) bool { + for _, vol := range vmSpec.Volumes { + if vol.PersistentVolumeClaim != nil { + return true + } + } + return false +} + +// HardwareVersionForPVCandPCIDevices returns a hardware version for VMs with PVCs and PCI devices(vGPUs/DDPIO devices) +// The hardware version is determined based on the below criteria: VMs with +// - Persistent Volume Claim (PVC) get the max(the image hardware version, minimum supported virtual hardware version for persistent volumes) +// - vGPUs/DDPIO devices get the max(the image hardware version, minimum supported virtual hardware version for PCI devices) +// - Both vGPU/DDPIO devices and PVCs get the max(the image hardware version, minimum supported virtual hardware version for PCI devices) +// - none of the above returns 0. +func HardwareVersionForPVCandPCIDevices(imageHWVersion int32, configSpec *types.VirtualMachineConfigSpec, hasPVC bool) int32 { + var configSpecHWVersion int32 + configSpecDevs := util.DevicesFromConfigSpec(configSpec) + + if len(util.SelectNvidiaVgpu(configSpecDevs)) > 0 || len(util.SelectDynamicDirectPathIO(configSpecDevs)) > 0 { + configSpecHWVersion = constants.MinSupportedHWVersionForPCIPassthruDevices + if imageHWVersion != 0 && imageHWVersion > constants.MinSupportedHWVersionForPCIPassthruDevices { + configSpecHWVersion = imageHWVersion + } + } else if hasPVC { + configSpecHWVersion = constants.MinSupportedHWVersionForPVC + if imageHWVersion != 0 && imageHWVersion > constants.MinSupportedHWVersionForPVC { + configSpecHWVersion = imageHWVersion + } + } + + return configSpecHWVersion +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go new file mode 100644 index 000000000..cf7fe642d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go @@ -0,0 +1,629 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + goctx "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vmUtilTests() { + + var ( + k8sClient client.Client + initObjects []client.Object + + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + vm := builder.DummyBasicVirtualMachineA2("test-vm", "dummy-ns") + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.WithValue(goctx.Background(), context.MaxDeployThreadsContextKey, 16), + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + JustBeforeEach(func() { + k8sClient = builder.NewFakeClient(initObjects...) + }) + + AfterEach(func() { + k8sClient = nil + initObjects = nil + }) + + Context("GetVirtualMachineClass", func() { + oldNamespacedVMClassFSSEnabledFunc := lib.IsNamespacedVMClassFSSEnabled + + // NOTE: As we currently have it, v1a2 must have this enabled. + When("WCP_Namespaced_VM_Class FSS is enabled", func() { + var ( + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + vmClass = builder.DummyVirtualMachineClass2A2("dummy-vm-class") + vmClass.Namespace = vmCtx.VM.Namespace + vmCtx.VM.Spec.ClassName = vmClass.Name + + lib.IsNamespacedVMClassFSSEnabled = func() bool { + return true + } + }) + + AfterEach(func() { + lib.IsNamespacedVMClassFSSEnabled = oldNamespacedVMClassFSSEnabledFunc + }) + + Context("VirtualMachineClass custom resource doesn't exist", func() { + It("Returns error and sets condition when VM Class does not exist", func() { + expectedErrMsg := fmt.Sprintf("virtualmachineclasses.vmoperator.vmware.com %q not found", vmCtx.VM.Spec.ClassName) + + _, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineConditionClassReady, "NotFound", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + Context("VirtualMachineClass custom resource exists", func() { + + When("Is not Ready", func() { + + BeforeEach(func() { + vmClass.Status.Ready = false + initObjects = append(initObjects, vmClass) + }) + + It("returns an error", func() { + _, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("VirtualMachineClass is not Ready")) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition( + vmopv1.VirtualMachineConditionClassReady, + "NotReady", + "VirtualMachineClass is not marked as Ready"), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("Is Ready", func() { + + BeforeEach(func() { + vmClass.Status.Ready = true + initObjects = append(initObjects, vmClass) + }) + + It("returns success", func() { + class, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(class).ToNot(BeNil()) + }) + }) + }) + }) + }) + + Context("GetVMImageStatusAndContentLibraryUUID", func() { + + // NOTE: As we currently have it, v1a2 must have this enabled. + When("WCPVMImageRegistry FSS is enabled", func() { + + var ( + nsVMImage *vmopv1.VirtualMachineImage + clusterVMImage *vmopv1.ClusterVirtualMachineImage + ) + + BeforeEach(func() { + nsVMImage = builder.DummyVirtualMachineImageA2("dummy-ns-vm-image") + nsVMImage.Namespace = vmCtx.VM.Namespace + conditions.MarkTrue(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + clusterVMImage = builder.DummyClusterVirtualMachineImageA2("dummy-cluster-vm-image") + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + + lib.IsWCPVMImageRegistryEnabled = func() bool { + return true + } + }) + + When("Neither cluster or namespace scoped VM image exists", func() { + + It("returns error and sets condition", func() { + _, _, _, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + expectedErrMsg := fmt.Sprintf("Failed to get the VM's image: %s", vmCtx.VM.Spec.ImageName) + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineConditionImageReady, "NotFound", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("VM image exists but the image is not ready", func() { + + BeforeEach(func() { + conditions.MarkFalse(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition, "NotReady", "") // XXX Until rollup condition + initObjects = append(initObjects, nsVMImage) + vmCtx.VM.Spec.ImageName = nsVMImage.Name + }) + + It("returns error and sets VM condition", func() { + _, _, _, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + expectedErrMsg := "VirtualMachineImage is not ready" + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition( + vmopv1.VirtualMachineConditionImageReady, "NotReady", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("Namespace scoped VirtualMachineImage exists and ready", func() { + BeforeEach(func() { + conditions.MarkTrue(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + initObjects = append(initObjects, nsVMImage) + vmCtx.VM.Spec.ImageName = nsVMImage.Name + }) + + It("returns success", func() { + imgObj, spec, status, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(imgObj).ToNot(BeNil()) + Expect(imgObj.GetObjectKind().GroupVersionKind().Kind).To(Equal("VirtualMachineImage")) + Expect(spec).ToNot(BeNil()) + Expect(status).ToNot(BeNil()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + }) + }) + + When("ClusterVirtualMachineImage exists and ready", func() { + BeforeEach(func() { + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + initObjects = append(initObjects, clusterVMImage) + vmCtx.VM.Spec.ImageName = clusterVMImage.Name + }) + + It("returns success", func() { + imgObj, spec, status, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(imgObj).ToNot(BeNil()) + Expect(imgObj.GetObjectKind().GroupVersionKind().Kind).To(Equal("ClusterVirtualMachineImage")) + Expect(spec).ToNot(BeNil()) + Expect(status).ToNot(BeNil()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + }) + }) + }) + }) + + Context("GetVirtualMachineBootstrap", func() { + const dataName = "dummy-vm-bootstrap-data" + const vAppDataName = "dummy-vm-bootstrap-vapp-data" + + var ( + bootstrapCM *corev1.ConfigMap + bootstrapSecret *corev1.Secret + bootstrapVAppCM *corev1.ConfigMap + ) + + BeforeEach(func() { + bootstrapCM = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string]string{ + "foo": "bar", + }, + } + + bootstrapSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string][]byte{ + "foo1": []byte("bar1"), + }, + } + + bootstrapVAppCM = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: vAppDataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string]string{ + "foo-vapp": "bar-vapp", + }, + } + }) + + When("Bootstrap via CloudInit", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, + } + vmCtx.VM.Spec.Bootstrap.CloudInit.RawCloudConfig.Name = dataName + }) + + It("return an error when resources does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM) + }) + + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo", "bar")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("Secret exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM, bootstrapSecret) + }) + + When("Prefers Secret over ConfigMap", func() { + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + // Prefer Secret over ConfigMap. + Expect(data).To(HaveKeyWithValue("foo1", "bar1")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + }) + }) + + When("Bootstrap via Sysprep", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + Sysprep: &vmopv1.VirtualMachineBootstrapSysprepSpec{}, + } + vmCtx.VM.Spec.Bootstrap.Sysprep.RawSysprep.Name = dataName + }) + + It("return an error when resource does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM) + }) + + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo", "bar")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("Secret exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM, bootstrapSecret) + }) + + When("Prefers Secret over ConfigMap", func() { + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo1", "bar1")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + }) + }) + + When("Bootstrap with vAppConfig", func() { + + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + VAppConfig: &vmopv1.VirtualMachineBootstrapVAppConfigSpec{}, + } + vmCtx.VM.Spec.Bootstrap.VAppConfig.RawProperties = vAppDataName + }) + + It("return an error when resource does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapVAppCM) + }) + + It("returns success", func() { + _, data, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo-vapp", "bar-vapp")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("vAppConfig with properties", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap.VAppConfig.Properties = []common.KeyValueOrSecretKeySelectorPair{ + { + Value: common.ValueOrSecretKeySelector{ + From: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: vAppDataName, + }, + Key: "foo-vapp", + }, + }, + }, + } + + It("returns success", func() { + _, _, exData, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(exData).To(HaveKey(vAppDataName)) + data := exData[vAppDataName] + Expect(data).To(HaveKeyWithValue("foo-vapp", "bar-vapp")) + }) + }) + }) + }) + }) + + Context("GetVMSetResourcePolicy", func() { + + var ( + vmResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + BeforeEach(func() { + vmResourcePolicy = &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-vm-rp", + Namespace: vmCtx.VM.Namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{Name: "fooRP"}, + Folder: "fooFolder", + }, + } + }) + + It("returns success when VM does not have SetResourcePolicy", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "" + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + + It("VM SetResourcePolicy does not exist", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "bogus" + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + + When("VM SetResourcePolicy exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, vmResourcePolicy) + vmCtx.VM.Spec.Reserved.ResourcePolicyName = vmResourcePolicy.Name + }) + + It("returns success", func() { + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).ToNot(BeNil()) + }) + }) + }) + + Context("AddInstanceStorageVolumes", func() { + + var ( + vmClass *vmopv1.VirtualMachineClass + ) + + expectInstanceStorageVolumes := func( + vm *vmopv1.VirtualMachine, + isStorage vmopv1.InstanceStorage) { + + ExpectWithOffset(1, isStorage.Volumes).ToNot(BeEmpty()) + isVolumes := instancestorage.FilterVolumes(vm) + ExpectWithOffset(1, isVolumes).To(HaveLen(len(isStorage.Volumes))) + + for _, isVol := range isStorage.Volumes { + found := false + + for idx, vol := range isVolumes { + claim := vol.PersistentVolumeClaim.InstanceVolumeClaim + if claim.StorageClass == isStorage.StorageClass && claim.Size == isVol.Size { + isVolumes = append(isVolumes[:idx], isVolumes[idx+1:]...) + found = true + break + } + } + + ExpectWithOffset(1, found).To(BeTrue(), "failed to find instance storage volume for %v", isVol) + } + } + + BeforeEach(func() { + vmClass = builder.DummyVirtualMachineClassA2() + }) + + When("InstanceStorage FFS is enabled", func() { + + It("VM Class does not contain instance storage volumes", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeFalse()) + Expect(instancestorage.FilterVolumes(vmCtx.VM)).To(BeEmpty()) + }) + + When("Instance Volume is added in VM Class", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.InstanceStorage = builder.DummyInstanceStorageA2() + }) + + It("Instance Volumes should be added", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + expectInstanceStorageVolumes(vmCtx.VM, vmClass.Spec.Hardware.InstanceStorage) + }) + + It("Instance Storage is already added to VM Spec.Volumes", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + + isVolumesBefore := instancestorage.FilterVolumes(vmCtx.VM) + expectInstanceStorageVolumes(vmCtx.VM, vmClass.Spec.Hardware.InstanceStorage) + + // Instance Storage is already configured, should not patch again + is = vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + isVolumesAfter := instancestorage.FilterVolumes(vmCtx.VM) + Expect(isVolumesAfter).To(HaveLen(len(isVolumesBefore))) + Expect(isVolumesAfter).To(Equal(isVolumesBefore)) + }) + }) + }) + }) + + Context("HasPVC", func() { + + Context("Spec has no PVC", func() { + It("will return false", func() { + spec := vmopv1.VirtualMachineSpec{} + Expect(vsphere.HasPVC(spec)).To(BeFalse()) + }) + }) + + Context("Spec has PVCs", func() { + It("will return true", func() { + spec := vmopv1.VirtualMachineSpec{ + Volumes: []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + }, + } + Expect(vsphere.HasPVC(spec)).To(BeTrue()) + }) + }) + }) + + Context("HardwareVersionForPVCandPCIDevices", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + imageHWVersion int32 + ) + + BeforeEach(func() { + imageHWVersion = 14 + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + }) + + It("ConfigSpec has PCI devices and VM spec has PVCs", func() { + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(17))) + }) + + It("ConfigSpec has PCI devices and VM spec has no PVCs", func() { + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, false)).To(Equal(int32(17))) + }) + + It("ConfigSpec has PCI devices, VM spec has PVCs image hardware version is higher than min supported HW version for PCI devices", func() { + imageHWVersion = 18 + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(18))) + }) + + It("VM spec has PVCs and config spec has no devices", func() { + configSpec = &types.VirtualMachineConfigSpec{} + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(15))) + }) + + It("VM spec has PVCs, config spec has no devices and image hardware version is higher than min supported PVC HW version", func() { + configSpec = &types.VirtualMachineConfigSpec{} + imageHWVersion = 16 + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(16))) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go b/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go new file mode 100644 index 000000000..ac42dce8a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() + +func vcSimTests() { + Describe("CPUFreq", cpuFreqTests) + Describe("InitOvfCacheAndLockPool", initOvfCacheAndLockPoolTests) + Describe("ResourcePolicyTests", resourcePolicyTests) + Describe("VirtualMachine", vmTests) + Describe("VirtualMachineE2E", vmE2ETests) + Describe("VirtualMachineUtilsTest", vmUtilTests) +} + +func TestVSphereProvider(t *testing.T) { + suite.Register(t, "VMProvider Tests", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/test/builder/fake.go b/test/builder/fake.go index 7aab65f77..5764ceac1 100644 --- a/test/builder/fake.go +++ b/test/builder/fake.go @@ -42,13 +42,15 @@ func KnownObjectTypes() []client.Object { &v1alpha2.VirtualMachineService{}, &v1alpha1.VirtualMachineClass{}, &v1alpha2.VirtualMachineClass{}, - &cnsv1alpha1.CnsNodeVmAttachment{}, &v1alpha1.VirtualMachinePublishRequest{}, &v1alpha2.VirtualMachinePublishRequest{}, &v1alpha1.ClusterVirtualMachineImage{}, &v1alpha2.ClusterVirtualMachineImage{}, &v1alpha1.VirtualMachineImage{}, &v1alpha2.VirtualMachineImage{}, + &cnsv1alpha1.CnsNodeVmAttachment{}, + &ncpv1alpha1.VirtualNetworkInterface{}, + &netopv1alpha1.NetworkInterface{}, } } diff --git a/test/builder/utila2.go b/test/builder/utila2.go index a36c8bc11..902587367 100644 --- a/test/builder/utila2.go +++ b/test/builder/utila2.go @@ -14,100 +14,26 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ) -func DummyVirtualMachineSetResourcePolicyA2() *vmopv1.VirtualMachineSetResourcePolicy { - return &vmopv1.VirtualMachineSetResourcePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-", - }, - Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ - ResourcePool: vmopv1.ResourcePoolSpec{ - Name: "dummy-resource-pool", - Reservations: vmopv1.VirtualMachineResourceSpec{ - Cpu: resource.MustParse("1Gi"), - Memory: resource.MustParse("2Gi"), - }, - Limits: vmopv1.VirtualMachineResourceSpec{ - Cpu: resource.MustParse("2Gi"), - Memory: resource.MustParse("4Gi"), - }, - }, - Folder: "dummy-folder", - ClusterModuleGroups: []string{"dummy-cluster-modules"}, - }, - } -} - -func DummyVirtualMachineServiceA2() *vmopv1.VirtualMachineService { - return &vmopv1.VirtualMachineService{ +func DummyVirtualMachineClass2A2(name string) *vmopv1.VirtualMachineClass { + return &vmopv1.VirtualMachineClass{ ObjectMeta: metav1.ObjectMeta{ - // Using image.GenerateName causes problems with unit tests - Name: fmt.Sprintf("test-%s", uuid.New()), + Name: name, }, - Spec: vmopv1.VirtualMachineServiceSpec{ - Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, - Ports: []vmopv1.VirtualMachineServicePort{ - { - Name: "dummy-port", - Protocol: "TCP", - Port: 42, - TargetPort: 4242, - }, - }, - Selector: map[string]string{ - "foo": "bar", + Spec: vmopv1.VirtualMachineClassSpec{ + Hardware: vmopv1.VirtualMachineClassHardware{ + Cpus: int64(2), + Memory: resource.MustParse("4Gi"), }, - }, - } -} - -func DummyVirtualMachineA2() *vmopv1.VirtualMachine { - return &vmopv1.VirtualMachine{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-", - Labels: map[string]string{}, - Annotations: map[string]string{}, - }, - Spec: vmopv1.VirtualMachineSpec{ - ImageName: DummyImageName, - ClassName: DummyClassName, - PowerState: vmopv1.VirtualMachinePowerStateOn, - Volumes: []vmopv1.VirtualMachineVolume{ - { - Name: DummyVolumeName, - VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ - PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ - PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: DummyPVCName, - }, - }, + Policies: vmopv1.VirtualMachineClassPolicies{ + Resources: vmopv1.VirtualMachineClassResources{ + Requests: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("1Gi"), + Memory: resource.MustParse("2Gi"), + }, + Limits: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("2Gi"), + Memory: resource.MustParse("4Gi"), }, - }, - }, - }, - } -} - -func DummyVirtualMachinePublishRequestA2(name, namespace, sourceName, itemName, clName string) *vmopv1.VirtualMachinePublishRequest { - return &vmopv1.VirtualMachinePublishRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Finalizers: []string{"virtualmachinepublishrequest.vmoperator.vmware.com"}, - }, - Spec: vmopv1.VirtualMachinePublishRequestSpec{ - Source: vmopv1.VirtualMachinePublishRequestSource{ - Name: sourceName, - APIVersion: "vmoperator.vmware.com/v1alpha2", - Kind: "VirtualMachine", - }, - Target: vmopv1.VirtualMachinePublishRequestTarget{ - Item: vmopv1.VirtualMachinePublishRequestTargetItem{ - Name: itemName, - }, - Location: vmopv1.VirtualMachinePublishRequestTargetLocation{ - Name: clName, - APIVersion: "imageregistry.vmware.com/v1alpha1", - Kind: "ContentLibrary", }, }, }, @@ -140,6 +66,20 @@ func DummyVirtualMachineClassA2() *vmopv1.VirtualMachineClass { } } +func DummyInstanceStorageA2() vmopv1.InstanceStorage { + return vmopv1.InstanceStorage{ + StorageClass: DummyStorageClassName, + Volumes: []vmopv1.InstanceStorageVolume{ + { + Size: resource.MustParse("256Gi"), + }, + { + Size: resource.MustParse("512Gi"), + }, + }, + } +} + func DummyInstanceStorageVirtualMachineVolumesA2() []vmopv1.VirtualMachineVolume { return []vmopv1.VirtualMachineVolume{ { @@ -173,6 +113,161 @@ func DummyInstanceStorageVirtualMachineVolumesA2() []vmopv1.VirtualMachineVolume } } +func DummyBasicVirtualMachineA2(name, namespace string) *vmopv1.VirtualMachine { + return &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: vmopv1.VirtualMachineSpec{ + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePowerStateOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + }, + } +} + +func DummyVirtualMachineA2() *vmopv1.VirtualMachine { + return &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: vmopv1.VirtualMachineSpec{ + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePowerStateOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + Volumes: []vmopv1.VirtualMachineVolume{ + { + Name: DummyVolumeName, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: DummyPVCName, + }, + }, + }, + }, + }, + /* TODO: Convert this if/as needed + NetworkInterfaces: []vmopv1.VirtualMachineNetworkInterface{ + { + NetworkName: DummyNetworkName, + NetworkType: "", + }, + { + NetworkName: DummyNetworkName + "-2", + NetworkType: "", + }, + }, + VmMetadata: &vmopv1.VirtualMachineMetadata{ + ConfigMapName: DummyMetadataCMName, + Transport: "ExtraConfig", + }, + */ + }, + } +} + +func AddDummyInstanceStorageVolumeA2(vm *vmopv1.VirtualMachine) { + vm.Spec.Volumes = append(vm.Spec.Volumes, DummyInstanceStorageVirtualMachineVolumesA2()...) +} + +func DummyVirtualMachineServiceA2() *vmopv1.VirtualMachineService { + return &vmopv1.VirtualMachineService{ + ObjectMeta: metav1.ObjectMeta{ + // Using image.GenerateName causes problems with unit tests + Name: fmt.Sprintf("test-%s", uuid.New()), + }, + Spec: vmopv1.VirtualMachineServiceSpec{ + Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, + Ports: []vmopv1.VirtualMachineServicePort{ + { + Name: "dummy-port", + Protocol: "TCP", + Port: 42, + TargetPort: 4242, + }, + }, + Selector: map[string]string{ + "foo": "bar", + }, + }, + } +} + +func DummyVirtualMachineSetResourcePolicyA2() *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: "dummy-resource-pool", + Reservations: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("1Gi"), + Memory: resource.MustParse("2Gi"), + }, + Limits: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("2Gi"), + Memory: resource.MustParse("4Gi"), + }, + }, + Folder: "dummy-folder", + ClusterModuleGroups: []string{"dummy-cluster-modules"}, + }, + } +} + +func DummyVirtualMachineSetResourcePolicy2A2(name, namespace string) *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: name, + }, + Folder: name, + }, + } +} + +func DummyVirtualMachinePublishRequestA2(name, namespace, sourceName, itemName, clName string) *vmopv1.VirtualMachinePublishRequest { + return &vmopv1.VirtualMachinePublishRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Finalizers: []string{"virtualmachinepublishrequest.vmoperator.vmware.com"}, + }, + Spec: vmopv1.VirtualMachinePublishRequestSpec{ + Source: vmopv1.VirtualMachinePublishRequestSource{ + Name: sourceName, + APIVersion: "vmoperator.vmware.com/v1alpha2", + Kind: "VirtualMachine", + }, + Target: vmopv1.VirtualMachinePublishRequestTarget{ + Item: vmopv1.VirtualMachinePublishRequestTargetItem{ + Name: itemName, + }, + Location: vmopv1.VirtualMachinePublishRequestTargetLocation{ + Name: clName, + APIVersion: "imageregistry.vmware.com/v1alpha1", + Kind: "ContentLibrary", + }, + }, + }, + } +} + func DummyVirtualMachineImageA2(imageName string) *vmopv1.VirtualMachineImage { return &vmopv1.VirtualMachineImage{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/builder/vcsim_test_context.go b/test/builder/vcsim_test_context.go index 9805a311f..daad84acc 100644 --- a/test/builder/vcsim_test_context.go +++ b/test/builder/vcsim_test_context.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Package builder is a comment just to silence the linter @@ -45,8 +45,11 @@ import ( _ "github.com/vmware/govmomi/vapi/cluster/simulator" _ "github.com/vmware/govmomi/vapi/simulator" - vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/conditions2" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/record" "github.com/vmware-tanzu/vm-operator/test/testutil" @@ -55,8 +58,9 @@ import ( type NetworkEnv string const ( - NetworkEnvVDS = NetworkEnv("vds") - NetworkEnvNSXT = NetworkEnv("nsx-t") + NetworkEnvVDS = NetworkEnv("vds") + NetworkEnvNSXT = NetworkEnv("nsx-t") + NetworkEnvNamed = NetworkEnv("named") NsxTLogicalSwitchUUID = "nsxt-dummy-ls-uuid" ) @@ -96,6 +100,9 @@ type VCSimTestConfig struct { // WithVMClassAsConfigDaynDate enables the WCP_VM_CLASS_AS_CONFIG_DAYNDATE FSS. WithVMClassAsConfigDaynDate bool + // WithV1A2 enables the VMServiceV1Alpha2FSS FSS. + WithV1A2 bool + // WithNetworkEnv is the network environment type. WithNetworkEnv NetworkEnv } @@ -137,6 +144,7 @@ type TestContextForVCSim struct { folder *object.Folder datastore *object.Datastore withFaultDomains bool + withV1A2 bool singleCCR *object.ClusterComputeResource azCCRs map[string][]*object.ClusterComputeResource @@ -160,7 +168,7 @@ func (s *TestSuite) NewTestContextForVCSim( ctx := newTestContextForVCSim(config, initObjects) - ctx.setupEnvFSS(config) + ctx.setupEnv(config) ctx.setupVCSim(config) ctx.setupContentLibrary(config) ctx.setupK8sConfig(config) @@ -180,6 +188,7 @@ func newTestContextForVCSim( PodNamespace: "vmop-pod-test", Recorder: fakeRecorder, withFaultDomains: config.WithFaultDomains, + withV1A2: config.WithV1A2, } if ctx.withFaultDomains { @@ -272,19 +281,22 @@ func (c *TestContextForVCSim) CreateWorkloadNamespace() WorkloadNamespaceInfo { Expect(c.Client.Update(c, ns)).To(Succeed()) } - if clID := c.ContentLibraryID; clID != "" { - csBinding := &vmopv1.ContentSourceBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - Namespace: ns.Name, - }, - ContentSourceRef: vmopv1.ContentSourceReference{ - APIVersion: vmopv1.SchemeGroupVersion.Group, - Kind: "ContentSource", - Name: clID, - }, + // Not the exact right FFS, but it's what we've plumbed and is otherwise implied. + if !c.withV1A2 { + if clID := c.ContentLibraryID; clID != "" { + csBinding := &v1alpha1.ContentSourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + Namespace: ns.Name, + }, + ContentSourceRef: v1alpha1.ContentSourceReference{ + APIVersion: v1alpha1.SchemeGroupVersion.Group, + Kind: "ContentSource", + Name: clID, + }, + } + Expect(c.Client.Create(c, csBinding)).To(Succeed()) } - Expect(c.Client.Create(c, csBinding)).To(Succeed()) } resourceQuota := &corev1.ResourceQuota{ @@ -313,10 +325,26 @@ func (c *TestContextForVCSim) CreateWorkloadNamespace() WorkloadNamespaceInfo { } } -// TODO: Get rid of runtime env checks so this isn't needed. -func (c *TestContextForVCSim) setupEnvFSS(config VCSimTestConfig) { +func (c *TestContextForVCSim) setupEnv(config VCSimTestConfig) { Expect(lib.SetVMOpNamespaceEnv(c.PodNamespace)).To(Succeed()) + switch config.WithNetworkEnv { + case NetworkEnvVDS: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeVDS)).To(Succeed()) + case NetworkEnvNSXT: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNSXT)).To(Succeed()) + case NetworkEnvNamed: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNamed)).To(Succeed()) + default: + Expect(os.Unsetenv(lib.NetworkProviderType)).To(Succeed()) + } + + v1a2 := "false" + if config.WithV1A2 { + v1a2 = "true" + } + Expect(os.Setenv(lib.VMServiceV1Alpha2FSS, v1a2)).To(Succeed()) + if config.WithContentLibrary { Expect(os.Setenv("CONTENT_API_WAIT_SECS", "1")).To(Succeed()) } @@ -465,32 +493,6 @@ func (c *TestContextForVCSim) setupContentLibrary(config VCSimTestConfig) { Expect(clID).ToNot(BeEmpty()) c.ContentLibraryID = clID - clProvider := &vmopv1.ContentLibraryProvider{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - }, - Spec: vmopv1.ContentLibraryProviderSpec{ - UUID: clID, - }, - } - Expect(c.Client.Create(c, clProvider)).To(Succeed()) - - cs := &vmopv1.ContentSource{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - }, - Spec: vmopv1.ContentSourceSpec{ - ProviderRef: vmopv1.ContentProviderReference{ - Name: clProvider.Name, - Kind: "ContentLibraryProvider", - }, - }, - } - Expect(c.Client.Create(c, cs)).To(Succeed()) - - Expect(controllerutil.SetOwnerReference(cs, clProvider, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Update(c, clProvider)).To(Succeed()) - libraryItem := library.Item{ Name: "test-image-ovf", Type: "ovf", @@ -498,12 +500,50 @@ func (c *TestContextForVCSim) setupContentLibrary(config VCSimTestConfig) { } c.ContentLibraryImageName = libraryItem.Name - vmImage := DummyVirtualMachineImage(c.ContentLibraryImageName) - Expect(controllerutil.SetOwnerReference(clProvider, vmImage, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Create(c, vmImage)).To(Succeed()) - - createContentLibraryItem(libMgr, libraryItem, + itemID := createContentLibraryItem(libMgr, libraryItem, path.Join(testutil.GetRootDirOrDie(), "images", "ttylinux-pc_i486-16.1.ovf")) + + // Not the exact right FFS, but it's what we've plumbed and is otherwise implied. + if c.withV1A2 { + // The image isn't quite as prod but sufficient for what we need here ATM. + clusterVMImage := DummyClusterVirtualMachineImageA2(c.ContentLibraryImageName) + clusterVMImage.Spec.ProviderRef.Kind = "ClusterContentLibraryItem" + Expect(c.Client.Create(c, clusterVMImage)).To(Succeed()) + clusterVMImage.Status.ProviderItemID = itemID + conditions2.MarkTrue(clusterVMImage, v1alpha2.VirtualMachineImageSyncedCondition) // TODO: Until we get rollup Ready condition + Expect(c.Client.Status().Update(c, clusterVMImage)).To(Succeed()) + + } else { + clProvider := &v1alpha1.ContentLibraryProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + }, + Spec: v1alpha1.ContentLibraryProviderSpec{ + UUID: clID, + }, + } + Expect(c.Client.Create(c, clProvider)).To(Succeed()) + + cs := &v1alpha1.ContentSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + }, + Spec: v1alpha1.ContentSourceSpec{ + ProviderRef: v1alpha1.ContentProviderReference{ + Name: clProvider.Name, + Kind: "ContentLibraryProvider", + }, + }, + } + Expect(c.Client.Create(c, cs)).To(Succeed()) + + Expect(controllerutil.SetOwnerReference(cs, clProvider, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Update(c, clProvider)).To(Succeed()) + + vmImage := DummyVirtualMachineImage(c.ContentLibraryImageName) + Expect(controllerutil.SetOwnerReference(clProvider, vmImage, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Create(c, vmImage)).To(Succeed()) + } } func (c *TestContextForVCSim) ContentLibraryItemTemplate(srcVMName, templateName string) { @@ -529,21 +569,30 @@ func (c *TestContextForVCSim) ContentLibraryItemTemplate(srcVMName, templateName }, } - _, err = vcenter.NewManager(c.RestClient).CreateTemplate(c, spec) + itemID, err := vcenter.NewManager(c.RestClient).CreateTemplate(c, spec) Expect(err).ToNot(HaveOccurred()) // Create the expected VirtualMachineImage for the template. - vmImage := DummyVirtualMachineImage(templateName) - cl := &vmopv1.ContentLibraryProvider{} - Expect(c.Client.Get(c, client.ObjectKey{Name: clID}, cl)).To(Succeed()) - Expect(controllerutil.SetOwnerReference(cl, vmImage, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Create(c, vmImage)).To(Succeed()) + if c.withV1A2 { + clusterVMImage := DummyClusterVirtualMachineImageA2(templateName) + clusterVMImage.Spec.ProviderRef.Kind = "ClusterContentLibraryItem" + Expect(c.Client.Create(c, clusterVMImage)).To(Succeed()) + clusterVMImage.Status.ProviderItemID = itemID + conditions2.MarkTrue(clusterVMImage, v1alpha2.VirtualMachineImageSyncedCondition) // TODO: Until we get rollup Ready condition + Expect(c.Client.Status().Update(c, clusterVMImage)).To(Succeed()) + } else { + vmImage := DummyVirtualMachineImage(templateName) + cl := &v1alpha1.ContentLibraryProvider{} + Expect(c.Client.Get(c, client.ObjectKey{Name: clID}, cl)).To(Succeed()) + Expect(controllerutil.SetOwnerReference(cl, vmImage, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Create(c, vmImage)).To(Succeed()) + } } func createContentLibraryItem( libMgr *library.Manager, libraryItem library.Item, - itemPath string) { + itemPath string) string { ctx := goctx.Background() @@ -590,6 +639,8 @@ func createContentLibraryItem( } Expect(uploadFunc(itemPath)).To(Succeed()) Expect(libMgr.CompleteLibraryItemUpdateSession(ctx, sessionID)).To(Succeed()) + + return itemID } func (c *TestContextForVCSim) setupK8sConfig(config VCSimTestConfig) { @@ -720,11 +771,42 @@ func (c *TestContextForVCSim) GetAZClusterComputes(azName string) []*object.Clus func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicy( name string, - nsInfo WorkloadNamespaceInfo) (*vmopv1.VirtualMachineSetResourcePolicy, *object.Folder) { + nsInfo WorkloadNamespaceInfo) (*v1alpha1.VirtualMachineSetResourcePolicy, *object.Folder) { + + ExpectWithOffset(1, c.withV1A2).To(BeFalse()) resourcePolicy := DummyVirtualMachineSetResourcePolicy2(name, nsInfo.Namespace) Expect(c.Client.Create(c, resourcePolicy)).To(Succeed()) + folder := c.createVirtualMachineSetResourcePolicyCommon( + resourcePolicy.Spec.ResourcePool.Name, + resourcePolicy.Spec.Folder.Name, + nsInfo) + + return resourcePolicy, folder +} + +func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicyA2( + name string, + nsInfo WorkloadNamespaceInfo) (*v1alpha2.VirtualMachineSetResourcePolicy, *object.Folder) { + + ExpectWithOffset(1, c.withV1A2).To(BeTrue()) + + resourcePolicy := DummyVirtualMachineSetResourcePolicy2A2(name, nsInfo.Namespace) + Expect(c.Client.Create(c, resourcePolicy)).To(Succeed()) + + folder := c.createVirtualMachineSetResourcePolicyCommon( + resourcePolicy.Spec.ResourcePool.Name, + resourcePolicy.Spec.Folder, + nsInfo) + + return resourcePolicy, folder +} + +func (c *TestContextForVCSim) createVirtualMachineSetResourcePolicyCommon( + rpName, folderName string, + nsInfo WorkloadNamespaceInfo) *object.Folder { + var rps []*object.ResourcePool if c.withFaultDomains { @@ -749,14 +831,14 @@ func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicy( nsRP, ok := objRef.(*object.ResourcePool) Expect(ok).To(BeTrue()) - _, err = nsRP.Create(c, resourcePolicy.Spec.ResourcePool.Name, types.DefaultResourceConfigSpec()) + _, err = nsRP.Create(c, rpName, types.DefaultResourceConfigSpec()) Expect(err).ToNot(HaveOccurred()) } - folder, err := nsInfo.Folder.CreateFolder(c, resourcePolicy.Spec.Folder.Name) + folder, err := nsInfo.Folder.CreateFolder(c, folderName) Expect(err).ToNot(HaveOccurred()) - return resourcePolicy, folder + return folder } func (c *TestContextForVCSim) GetVMFromMoID(moID string) *object.VirtualMachine { From d424e4f79b881ab90f81b64de99df1412a2902f9 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 6 Sep 2023 15:46:01 -0500 Subject: [PATCH 4/5] Enable the v1a2 webhooks with the FFS is enabled We'll likely need the existing v1a1 mutation webhooks for some types - definitely for the VM - but that can wait to be addressed later. However, we will not need both v1a1 and v1a2 validation webhooks since the v1a2 hooks will cover v1a1 after version conversion. Correct some package names from 9869e812 --- config/webhook/manifests.yaml | 72 +++++++++---------- webhooks/virtualmachine/webhooks.go | 13 +++- .../virtualmachineclass/v1alpha2/webhooks.go | 2 +- webhooks/virtualmachineclass/webhooks.go | 6 ++ .../v1alpha2/webhooks.go | 2 +- .../virtualmachinepublishrequest/webhooks.go | 5 ++ webhooks/virtualmachineservice/webhooks.go | 5 ++ .../webhooks.go | 5 ++ 8 files changed, 70 insertions(+), 40 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index faaceb654..8b124ee10 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -103,19 +103,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachine failurePolicy: Fail - name: default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - virtualmachineclasses + - virtualmachines sideEffects: None - admissionReviewVersions: - v1 @@ -124,9 +124,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass failurePolicy: Fail - name: default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -136,7 +136,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachinepublishrequests + - virtualmachineclasses sideEffects: None - admissionReviewVersions: - v1 @@ -145,19 +145,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineclass failurePolicy: Fail - name: default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachineclass.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - virtualmachineservices + - virtualmachineclasses sideEffects: None - admissionReviewVersions: - v1 @@ -166,9 +166,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest failurePolicy: Fail - name: default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -178,7 +178,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachinesetresourcepolicies + - virtualmachinepublishrequests sideEffects: None - admissionReviewVersions: - v1 @@ -187,19 +187,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinepublishrequest failurePolicy: Fail - name: default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachinepublishrequest.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - webconsolerequests + - virtualmachinepublishrequests sideEffects: None - admissionReviewVersions: - v1 @@ -208,19 +208,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachine + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice failurePolicy: Fail - name: default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachines + - virtualmachineservices sideEffects: None - admissionReviewVersions: - v1 @@ -229,9 +229,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineclass + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineservice failurePolicy: Fail - name: default.validating.virtualmachineclass.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachineservice.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -241,7 +241,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachineclasses + - virtualmachineservices sideEffects: None - admissionReviewVersions: - v1 @@ -250,19 +250,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinepublishrequest + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy failurePolicy: Fail - name: default.validating.virtualmachinepublishrequest.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachinepublishrequests + - virtualmachinesetresourcepolicies sideEffects: None - admissionReviewVersions: - v1 @@ -271,9 +271,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineservice + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinesetresourcepolicy failurePolicy: Fail - name: default.validating.virtualmachineservice.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachinesetresourcepolicy.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -283,7 +283,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachineservices + - virtualmachinesetresourcepolicies sideEffects: None - admissionReviewVersions: - v1 @@ -292,17 +292,17 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinesetresourcepolicy + path: /default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest failurePolicy: Fail - name: default.validating.virtualmachinesetresourcepolicy.v1alpha2.vmoperator.vmware.com + name: default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachinesetresourcepolicies + - webconsolerequests sideEffects: None diff --git a/webhooks/virtualmachine/webhooks.go b/webhooks/virtualmachine/webhooks.go index 49bf76683..83370a430 100644 --- a/webhooks/virtualmachine/webhooks.go +++ b/webhooks/virtualmachine/webhooks.go @@ -9,12 +9,21 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := v1alpha1.AddToManager(ctx, mgr); err != nil { - return errors.Wrap(err, "failed to initialize v1alpha1 webhooks") + if lib.IsVMServiceV1Alpha2FSSEnabled() { + // TODO: We'll likely still need the v1a1 mutation wehbook (at least some limited version of it) + if err := v1alpha2.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize v1alpha2 webhooks") + } + } else { + if err := v1alpha1.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize v1alpha1 webhooks") + } } return nil diff --git a/webhooks/virtualmachineclass/v1alpha2/webhooks.go b/webhooks/virtualmachineclass/v1alpha2/webhooks.go index f727028ec..226641b85 100644 --- a/webhooks/virtualmachineclass/v1alpha2/webhooks.go +++ b/webhooks/virtualmachineclass/v1alpha2/webhooks.go @@ -1,7 +1,7 @@ // Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package virtualmachineclass +package v1alpha2 import ( "github.com/pkg/errors" diff --git a/webhooks/virtualmachineclass/webhooks.go b/webhooks/virtualmachineclass/webhooks.go index 9f6063a6c..4611b5248 100644 --- a/webhooks/virtualmachineclass/webhooks.go +++ b/webhooks/virtualmachineclass/webhooks.go @@ -7,9 +7,15 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineclass/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineclass/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } + return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go b/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go index 213223611..8c2581a5a 100644 --- a/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go +++ b/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go @@ -1,7 +1,7 @@ // Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package virtualmachinepublishrequest +package v1alpha2 import ( "github.com/pkg/errors" diff --git a/webhooks/virtualmachinepublishrequest/webhooks.go b/webhooks/virtualmachinepublishrequest/webhooks.go index 5c3759fce..91a84119f 100644 --- a/webhooks/virtualmachinepublishrequest/webhooks.go +++ b/webhooks/virtualmachinepublishrequest/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinepublishrequest/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinepublishrequest/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachineservice/webhooks.go b/webhooks/virtualmachineservice/webhooks.go index 440b05332..c13880752 100644 --- a/webhooks/virtualmachineservice/webhooks.go +++ b/webhooks/virtualmachineservice/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineservice/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineservice/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachinesetresourcepolicy/webhooks.go b/webhooks/virtualmachinesetresourcepolicy/webhooks.go index 6e064b195..8d123c0bf 100644 --- a/webhooks/virtualmachinesetresourcepolicy/webhooks.go +++ b/webhooks/virtualmachinesetresourcepolicy/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinesetresourcepolicy/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinesetresourcepolicy/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } From 865f40785535962fc4cbc157bec1291cb1c90daa Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Thu, 28 Sep 2023 14:45:53 -0500 Subject: [PATCH 5/5] Hook up ctrl-runtime webhooks when the v1a2 FFS is enabled This is used for the version conversion wehbooks. This can also support validation and mutation webhooks but our own webhook framework predates ctrl-runtime support for that so that is an improvement for later. --- main.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/main.go b/main.go index c40785abc..65958e50e 100644 --- a/main.go +++ b/main.go @@ -18,9 +18,12 @@ import ( klog "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/controllers" "github.com/vmware-tanzu/vm-operator/pkg" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/manager" "github.com/vmware-tanzu/vm-operator/webhooks" @@ -265,6 +268,12 @@ func main() { return err } + if lib.IsVMServiceV1Alpha2FSSEnabled() { + if err := addConversionWebhooksToManager(ctx, mgr); err != nil { + return err + } + } + return webhooks.AddToManager(ctx, mgr) } @@ -296,6 +305,57 @@ func main() { } } +// addConversionWebhooksToManager adds the ctrl-runtime managed webhooks. We just use these +// for version conversion, but they can also do mutation and validation webhook callbacks +// instead of our separate webhooks. +func addConversionWebhooksToManager(_ *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if err := (&v1alpha1.VirtualMachine{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineClass{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.ClusterVirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachinePublishRequest{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineService{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineSetResourcePolicy{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + + if err := (&v1alpha2.VirtualMachine{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineClass{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.ClusterVirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachinePublishRequest{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineService{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineSetResourcePolicy{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + + return nil +} + func configureWebhookTLS(opts *webhook.Options) { tlsCfgFunc := func(cfg *tls.Config) { cfg.MinVersion = tls.VersionTLS12