feat: add vgpu to vm resource

Add a new option `shared-pci-device-id` to the `virtual machine` resource. This enables users to 1 or more vGPU to the VM as an update to the resource or during the clone operation.
hashicorp · Apr 29, 2024 · bdf3e18 · bdf3e18
1 parent 1895bc1
commit bdf3e18
Show file tree

Hide file tree

Showing 3 changed files with 452 additions and 252 deletions.
diff --git a/vsphere/internal/virtualdevice/virtual_machine_device_subresource.go b/vsphere/internal/virtualdevice/virtual_machine_device_subresource.go
@@ -1018,7 +1018,7 @@ func (c *pciApplyConfig) modifyVirtualPciDevices(devList *schema.Set, op types.V
 }
 
 // PciPassthroughApplyOperation checks for changes in a virtual machine's
-// PCI passthrough devices and creates config specs to apply apply to the
+// PCI passthrough devices and creates config specs to apply to the
 // virtual machine.
 func PciPassthroughApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
 	old, newValue := d.GetChange("pci_device_id")
@@ -1095,3 +1095,148 @@ func PciPassthroughPostCloneOperation(d *schema.ResourceData, c *govmomi.Client,
 	}
 	return applyConfig.VirtualDevice, applyConfig.Spec, nil
 }
+
+// modifyVirtualSharedPciDevices will take a list of devices and an operation and
+// will create the appropriate config spec.
+func (c *pciApplyConfig) modifyVirtualSharedPciDevices(devList *schema.Set, op types.VirtualDeviceConfigSpecOperation) error {
+	log.Printf("VirtualMachine: Creating Shared PCI device specs %v", op)
+	for _, devId := range devList.List() {
+		log.Printf("[DEBUG] modifyVirtualSharedPciDevices: Appending %v spec for %s", op, devId.(string))
+
+		dev := &types.VirtualPCIPassthrough{
+			VirtualDevice: types.VirtualDevice{
+				DynamicData: types.DynamicData{},
+				Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{
+					Vgpu: devId.(string),
+				},
+			},
+		}
+
+		vm, err := virtualmachine.FromUUID(c.Client, c.ResourceData.Id())
+		if err != nil {
+			return err
+		}
+
+		vprops, err := virtualmachine.Properties(vm)
+		if err != nil {
+			return err
+		}
+
+		// This will only find a device for delete operations.
+		for _, vmDevP := range vprops.Config.Hardware.Device {
+			if vmDev, ok := vmDevP.(*types.VirtualPCIPassthrough); ok {
+				if vmDev.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo).Vgpu == devId {
+					dev = vmDev
+				}
+			}
+		}
+
+		dspec, err := object.VirtualDeviceList{dev}.ConfigSpec(op)
+		if err != nil {
+			return err
+		}
+
+		c.Spec = append(c.Spec, dspec...)
+		c.VirtualDevice = applyDeviceChange(c.VirtualDevice, dspec)
+	}
+	log.Printf("VirtualMachine: Shared PCI device specs created")
+	return nil
+}
+
+// SharedPciApplyOperation checks for changes in a virtual machine's
+// Shared PCI device and creates config specs to apply to the
+// virtual machine.
+func SharedPciApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
+	log.Printf("[DEBUG] SharedPciApplyOperation: Looking for shared PCI device changes")
+
+	// Get current (old) and new devices
+	old, newValue := d.GetChange("shared_pci_device_id")
+	oldDevIds := old.(*schema.Set)
+	newDevIds := newValue.(*schema.Set)
+
+	// Compare
+	delDevs := oldDevIds.Difference(newDevIds)
+	addDevs := newDevIds.Difference(oldDevIds)
+
+	// Create base apply config
+	applyConfig := &pciApplyConfig{
+		Client:        c,
+		ResourceData:  d,
+		Spec:          []types.BaseVirtualDeviceConfigSpec{},
+		VirtualDevice: l,
+	}
+
+	// If there are no changes, return as is
+	if addDevs.Len() == 0 && delDevs.Len() == 0 {
+		log.Printf("[DEBUG] SharedPciApplyOperation: No shared PCI device additions/deletions")
+		return applyConfig.VirtualDevice, applyConfig.Spec, nil
+	}
+
+	// Set reboot
+	_ = d.Set("reboot_required", true)
+
+	// Add new Shared PCI devices
+	log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device additions",
+		addDevs.Len())
+	err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Remove deleted Shared PCI devices
+	log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device deletions",
+		delDevs.Len())
+	err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return applyConfig.VirtualDevice, applyConfig.Spec, nil
+}
+
+// SharedPciPostCloneOperation normalizes the Shared PCI devices
+// on a newly-cloned virtual machine and outputs any necessary device change
+// operations. It also sets the state in advance of the post-create read.
+func SharedPciPostCloneOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
+	log.Printf("[DEBUG] SharedPCIPostCloneOperation: Looking for shared PCI device changes post-clone")
+
+	// Get current (old) and new devices
+	old, newValue := d.GetChange("shared_pci_device_id")
+	oldDevIds := old.(*schema.Set)
+	newDevIds := newValue.(*schema.Set)
+
+	// Compare
+	delDevs := oldDevIds.Difference(newDevIds)
+	addDevs := newDevIds.Difference(oldDevIds)
+
+	// Create base apply config
+	applyConfig := &pciApplyConfig{
+		Client:        c,
+		ResourceData:  d,
+		Spec:          []types.BaseVirtualDeviceConfigSpec{},
+		VirtualDevice: l,
+	}
+
+	// If there are no changes, return as is
+	if addDevs.Len() <= 0 && delDevs.Len() <= 0 {
+		log.Printf("[DEBUG] SharedPCIPostCloneOperation: No shared PCI device additions/deletions post-clone")
+		return applyConfig.VirtualDevice, applyConfig.Spec, nil
+	}
+
+	// Add new Shared PCI devices
+	log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device additions post-clone",
+		addDevs.Len())
+	err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Remove deleted Shared PCI devices
+	log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device deletions post-clone",
+		delDevs.Len())
+	err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove)
+	if err != nil {
+		return nil, nil, err
+	}
+	return applyConfig.VirtualDevice, applyConfig.Spec, nil
+}
diff --git a/vsphere/resource_vsphere_virtual_machine.go b/vsphere/resource_vsphere_virtual_machine.go
@@ -7,14 +7,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/guestoscustomizations"
 	"log"
 	"net"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/guestoscustomizations"
+
 	"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/contentlibrary"
 	"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/ovfdeploy"
 
@@ -244,6 +245,12 @@ func resourceVSphereVirtualMachine() *schema.Resource {
 			Description: "A list of PCI passthrough devices",
 			Elem:        &schema.Schema{Type: schema.TypeString},
 		},
+		"shared_pci_device_id": {
+			Type:        schema.TypeSet,
+			Optional:    true,
+			Description: "A list of Shared PCI passthrough device, 'grid_rtx8000-8q'",
+			Elem:        &schema.Schema{Type: schema.TypeString},
+		},
 		"clone": {
 			Type:        schema.TypeList,
 			Optional:    true,
@@ -530,20 +537,34 @@ func resourceVSphereVirtualMachineRead(d *schema.ResourceData, meta interface{})
 
 	// Read the virtual machine PCI passthrough devices
 	var pciDevs []string
+	var sharedPciDevs []string
 	for _, dev := range vprops.Config.Hardware.Device {
 		if pci, ok := dev.(*types.VirtualPCIPassthrough); ok {
-			if pciBacking, ok := pci.Backing.(*types.VirtualPCIPassthroughDeviceBackingInfo); ok {
-				devId := pciBacking.Id
+			switch t := pci.Backing.(type) {
+			case *types.VirtualPCIPassthroughDeviceBackingInfo:
+				devId := t.Id
 				pciDevs = append(pciDevs, devId)
-			} else {
-				log.Printf("[DEBUG] %s: PCI passthrough device %q has no backing ID", resourceVSphereVirtualMachineIDString(d), pci.GetVirtualDevice().DeviceInfo.GetDescription())
+				log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T",
+					vm.InventoryPath, devId, pci.Backing)
+			case *types.VirtualPCIPassthroughVmiopBackingInfo:
+				dev := t.Vgpu
+				sharedPciDevs = append(sharedPciDevs, dev)
+				log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T",
+					vm.InventoryPath, dev, pci.Backing)
+			default:
+				log.Printf("[WARN] Ignoring VM %q VirtualPCIPassthrough device with backing type of %T",
+					vm.InventoryPath, pci.Backing)
 			}
 		}
 	}
 	err = d.Set("pci_device_id", pciDevs)
 	if err != nil {
 		return err
 	}
+	err = d.Set("shared_pci_device_id", sharedPciDevs)
+	if err != nil {
+		return err
+	}
 
 	// Perform pending device read operations.
 	devices := object.VirtualDeviceList(vprops.Config.Hardware.Device)
@@ -1685,6 +1706,17 @@ func resourceVSphereVirtualMachinePostDeployChanges(d *schema.ResourceData, meta
 		)
 	}
 	cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...)
+	// Shared PCI devices
+	devices, delta, err = virtualdevice.SharedPciPostCloneOperation(d, client, devices)
+	if err != nil {
+		return resourceVSphereVirtualMachineRollbackCreate(
+			d,
+			meta,
+			vm,
+			fmt.Errorf("error processing shared PCI device changes post-clone: %s", err),
+		)
+	}
+	cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...)
 	log.Printf("[DEBUG] %s: Final device list: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceListString(devices))
 	log.Printf("[DEBUG] %s: Final device change cfgSpec: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceChangeString(cfgSpec.DeviceChange))
 
@@ -1988,6 +2020,12 @@ func applyVirtualDevices(d *schema.ResourceData, c *govmomi.Client, l object.Vir
 		return nil, err
 	}
 	spec = virtualdevice.AppendDeviceChangeSpec(spec, delta...)
+	// Shared PCI device
+	l, delta, err = virtualdevice.SharedPciApplyOperation(d, c, l)
+	if err != nil {
+		return nil, err
+	}
+	spec = virtualdevice.AppendDeviceChangeSpec(spec, delta...)
 	log.Printf("[DEBUG] %s: Final device list: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceListString(l))
 	log.Printf("[DEBUG] %s: Final device change spec: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceChangeString(spec))
 	return spec, nil