Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for vgpu #2113

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: add support for vgpu #2113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 146 additions & 1 deletion vsphere/internal/virtualdevice/virtual_machine_device_subresource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1018,7 +1018,7 @@ func (c *pciApplyConfig) modifyVirtualPciDevices(devList *schema.Set, op types.V
}

// PciPassthroughApplyOperation checks for changes in a virtual machine's
// PCI passthrough devices and creates config specs to apply apply to the
// PCI passthrough devices and creates config specs to apply to the
// virtual machine.
func PciPassthroughApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
old, newValue := d.GetChange("pci_device_id")
Expand Down Expand Up @@ -1095,3 +1095,148 @@ func PciPassthroughPostCloneOperation(d *schema.ResourceData, c *govmomi.Client,
}
return applyConfig.VirtualDevice, applyConfig.Spec, nil
}

// modifyVirtualSharedPciDevices will take a list of devices and an operation and
// will create the appropriate config spec.
func (c *pciApplyConfig) modifyVirtualSharedPciDevices(devList *schema.Set, op types.VirtualDeviceConfigSpecOperation) error {
log.Printf("VirtualMachine: Creating Shared PCI device specs %v", op)
for _, devId := range devList.List() {
log.Printf("[DEBUG] modifyVirtualSharedPciDevices: Appending %v spec for %s", op, devId.(string))

dev := &types.VirtualPCIPassthrough{
VirtualDevice: types.VirtualDevice{
DynamicData: types.DynamicData{},
Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{
Vgpu: devId.(string),
},
},
}

vm, err := virtualmachine.FromUUID(c.Client, c.ResourceData.Id())
if err != nil {
return err
}

vprops, err := virtualmachine.Properties(vm)
if err != nil {
return err
}

// This will only find a device for delete operations.
for _, vmDevP := range vprops.Config.Hardware.Device {
if vmDev, ok := vmDevP.(*types.VirtualPCIPassthrough); ok {
if vmDev.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo).Vgpu == devId {
dev = vmDev
}
}
}

dspec, err := object.VirtualDeviceList{dev}.ConfigSpec(op)
if err != nil {
return err
}

c.Spec = append(c.Spec, dspec...)
c.VirtualDevice = applyDeviceChange(c.VirtualDevice, dspec)
}
log.Printf("VirtualMachine: Shared PCI device specs created")
return nil
}

// SharedPciApplyOperation checks for changes in a virtual machine's
// Shared PCI device and creates config specs to apply to the
// virtual machine.
func SharedPciApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
log.Printf("[DEBUG] SharedPciApplyOperation: Looking for shared PCI device changes")

// Get current (old) and new devices
old, newValue := d.GetChange("shared_pci_device_id")
oldDevIds := old.(*schema.Set)
newDevIds := newValue.(*schema.Set)

// Compare
delDevs := oldDevIds.Difference(newDevIds)
addDevs := newDevIds.Difference(oldDevIds)

// Create base apply config
applyConfig := &pciApplyConfig{
Client: c,
ResourceData: d,
Spec: []types.BaseVirtualDeviceConfigSpec{},
VirtualDevice: l,
}

// If there are no changes, return as is
if addDevs.Len() == 0 && delDevs.Len() == 0 {
log.Printf("[DEBUG] SharedPciApplyOperation: No shared PCI device additions/deletions")
return applyConfig.VirtualDevice, applyConfig.Spec, nil
}

// Set reboot
_ = d.Set("reboot_required", true)

// Add new Shared PCI devices
log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device additions",
addDevs.Len())
err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd)
if err != nil {
return nil, nil, err
}

// Remove deleted Shared PCI devices
log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device deletions",
delDevs.Len())
err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove)
if err != nil {
return nil, nil, err
}

return applyConfig.VirtualDevice, applyConfig.Spec, nil
}

// SharedPciPostCloneOperation normalizes the Shared PCI devices
// on a newly-cloned virtual machine and outputs any necessary device change
// operations. It also sets the state in advance of the post-create read.
func SharedPciPostCloneOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) {
log.Printf("[DEBUG] SharedPCIPostCloneOperation: Looking for shared PCI device changes post-clone")

// Get current (old) and new devices
old, newValue := d.GetChange("shared_pci_device_id")
oldDevIds := old.(*schema.Set)
newDevIds := newValue.(*schema.Set)

// Compare
delDevs := oldDevIds.Difference(newDevIds)
addDevs := newDevIds.Difference(oldDevIds)

// Create base apply config
applyConfig := &pciApplyConfig{
Client: c,
ResourceData: d,
Spec: []types.BaseVirtualDeviceConfigSpec{},
VirtualDevice: l,
}

// If there are no changes, return as is
if addDevs.Len() <= 0 && delDevs.Len() <= 0 {
log.Printf("[DEBUG] SharedPCIPostCloneOperation: No shared PCI device additions/deletions post-clone")
return applyConfig.VirtualDevice, applyConfig.Spec, nil
}

// Add new Shared PCI devices
log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device additions post-clone",
addDevs.Len())
err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd)
if err != nil {
return nil, nil, err
}

// Remove deleted Shared PCI devices
log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device deletions post-clone",
delDevs.Len())
err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove)
if err != nil {
return nil, nil, err
}
return applyConfig.VirtualDevice, applyConfig.Spec, nil
}
48 changes: 43 additions & 5 deletions vsphere/resource_vsphere_virtual_machine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@ import (
"context"
"errors"
"fmt"
"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/guestoscustomizations"
"log"
"net"
"os"
"path/filepath"
"strings"
"time"

"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/guestoscustomizations"

"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/contentlibrary"
"github.com/hashicorp/terraform-provider-vsphere/vsphere/internal/helper/ovfdeploy"

Expand Down Expand Up @@ -244,6 +245,12 @@ func resourceVSphereVirtualMachine() *schema.Resource {
Description: "A list of PCI passthrough devices",
Elem: &schema.Schema{Type: schema.TypeString},
},
"shared_pci_device_id": {
Type: schema.TypeSet,
Optional: true,
Description: "A list of Shared PCI passthrough device, 'grid_rtx8000-8q'",
Elem: &schema.Schema{Type: schema.TypeString},
},
"clone": {
Type: schema.TypeList,
Optional: true,
Expand Down Expand Up @@ -530,20 +537,34 @@ func resourceVSphereVirtualMachineRead(d *schema.ResourceData, meta interface{})

// Read the virtual machine PCI passthrough devices
var pciDevs []string
var sharedPciDevs []string
for _, dev := range vprops.Config.Hardware.Device {
if pci, ok := dev.(*types.VirtualPCIPassthrough); ok {
if pciBacking, ok := pci.Backing.(*types.VirtualPCIPassthroughDeviceBackingInfo); ok {
devId := pciBacking.Id
switch t := pci.Backing.(type) {
case *types.VirtualPCIPassthroughDeviceBackingInfo:
devId := t.Id
pciDevs = append(pciDevs, devId)
} else {
log.Printf("[DEBUG] %s: PCI passthrough device %q has no backing ID", resourceVSphereVirtualMachineIDString(d), pci.GetVirtualDevice().DeviceInfo.GetDescription())
log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T",
vm.InventoryPath, devId, pci.Backing)
case *types.VirtualPCIPassthroughVmiopBackingInfo:
dev := t.Vgpu
sharedPciDevs = append(sharedPciDevs, dev)
log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T",
vm.InventoryPath, dev, pci.Backing)
default:
log.Printf("[WARN] Ignoring VM %q VirtualPCIPassthrough device with backing type of %T",
vm.InventoryPath, pci.Backing)
}
}
}
err = d.Set("pci_device_id", pciDevs)
if err != nil {
return err
}
err = d.Set("shared_pci_device_id", sharedPciDevs)
if err != nil {
return err
}

// Perform pending device read operations.
devices := object.VirtualDeviceList(vprops.Config.Hardware.Device)
Expand Down Expand Up @@ -1685,6 +1706,17 @@ func resourceVSphereVirtualMachinePostDeployChanges(d *schema.ResourceData, meta
)
}
cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...)
// Shared PCI devices
devices, delta, err = virtualdevice.SharedPciPostCloneOperation(d, client, devices)
if err != nil {
return resourceVSphereVirtualMachineRollbackCreate(
d,
meta,
vm,
fmt.Errorf("error processing shared PCI device changes post-clone: %s", err),
)
}
cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...)
log.Printf("[DEBUG] %s: Final device list: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceListString(devices))
log.Printf("[DEBUG] %s: Final device change cfgSpec: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceChangeString(cfgSpec.DeviceChange))

Expand Down Expand Up @@ -1988,6 +2020,12 @@ func applyVirtualDevices(d *schema.ResourceData, c *govmomi.Client, l object.Vir
return nil, err
}
spec = virtualdevice.AppendDeviceChangeSpec(spec, delta...)
// Shared PCI device
l, delta, err = virtualdevice.SharedPciApplyOperation(d, c, l)
if err != nil {
return nil, err
}
spec = virtualdevice.AppendDeviceChangeSpec(spec, delta...)
log.Printf("[DEBUG] %s: Final device list: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceListString(l))
log.Printf("[DEBUG] %s: Final device change spec: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceChangeString(spec))
return spec, nil
Expand Down
Loading
Loading