From 55c9b2f13c57f275008012c96710eb0de33b1909 Mon Sep 17 00:00:00 2001 From: mristok Date: Wed, 10 Jan 2024 13:16:00 -0600 Subject: [PATCH] feat: add vgpu to vm resource --- go.mod | 2 +- go.sum | 3 +- .../virtual_machine_device_subresource.go | 167 +++++++++++++----- vsphere/resource_vsphere_file.go | 7 +- vsphere/resource_vsphere_virtual_machine.go | 49 +++-- website/docs/r/virtual_machine.html.markdown | 20 ++- 6 files changed, 178 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index 0739fe663..526ec8dc7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/agext/levenshtein v1.2.2 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/fatih/color v1.13.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.6.0 // indirect diff --git a/go.sum b/go.sum index 7647526c5..aa07c69bd 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,9 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/vsphere/internal/virtualdevice/virtual_machine_device_subresource.go b/vsphere/internal/virtualdevice/virtual_machine_device_subresource.go index 1c5e576f0..429031834 100644 --- a/vsphere/internal/virtualdevice/virtual_machine_device_subresource.go +++ b/vsphere/internal/virtualdevice/virtual_machine_device_subresource.go @@ -1013,7 +1013,7 @@ func (c *pciApplyConfig) modifyVirtualPciDevices(devList *schema.Set, op types.V } // PciPassthroughApplyOperation checks for changes in a virtual machine's -// PCI passthrough devices and creates config specs to apply apply to the +// PCI passthrough devices and creates config specs to apply to the // virtual machine. func PciPassthroughApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) { old, newValue := d.GetChange("pci_device_id") @@ -1091,62 +1091,147 @@ func PciPassthroughPostCloneOperation(d *schema.ResourceData, c *govmomi.Client, return applyConfig.VirtualDevice, applyConfig.Spec, nil } -// SharedPciPassthroughApplyOperation checks for changes in a virtual machine's -// Shared PCI passthrough device and creates config specs to apply apply to the -// virtual machine. -func SharedPciPassthroughApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) { - old, new := d.GetChange("shared_pci_device_id") - oldDevId := old.(string) - newDevId := new.(string) +// modifyVirtualSharedPciDevices will take a list of devices and an operation and +// will create the appropriate config spec. +func (c *pciApplyConfig) modifyVirtualSharedPciDevices(devList *schema.Set, op types.VirtualDeviceConfigSpecOperation) error { + log.Printf("VirtualMachine: Creating Shared PCI device specs %v", op) + for _, devId := range devList.List() { + log.Printf("[DEBUG] modifyVirtualSharedPciDevices: Appending %v spec for %s", op, devId.(string)) - var specs []types.BaseVirtualDeviceConfigSpec - if oldDevId == newDevId { - return l, specs, nil - } + dev := &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + DynamicData: types.DynamicData{}, + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: devId.(string), + }, + }, + } - if oldDevId != "" { - vm, err := virtualmachine.FromUUID(c, d.Id()) + vm, err := virtualmachine.FromUUID(c.Client, c.ResourceData.Id()) if err != nil { - return nil, nil, err + return err } + vprops, err := virtualmachine.Properties(vm) if err != nil { - return nil, nil, err + return err } + // This will only find a device for delete operations. for _, vmDevP := range vprops.Config.Hardware.Device { if vmDev, ok := vmDevP.(*types.VirtualPCIPassthrough); ok { - if vmDev.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo).Vgpu == oldDevId { - dspec, err := object.VirtualDeviceList{vmDev}.ConfigSpec(types.VirtualDeviceConfigSpecOperationRemove) - if err != nil { - return nil, nil, err - } - specs = append(specs, dspec...) - - l = applyDeviceChange(l, dspec) - d.Set("reboot_required", true) + if vmDev.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo).Vgpu == devId { + dev = vmDev } } } - } - if newDevId != "" { - dev := &types.VirtualPCIPassthrough{ - VirtualDevice: types.VirtualDevice{ - DynamicData: types.DynamicData{}, - Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ - Vgpu: newDevId, - }, - }, - } - dspec, err := object.VirtualDeviceList{dev}.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd) + dspec, err := object.VirtualDeviceList{dev}.ConfigSpec(op) if err != nil { - return nil, nil, err + return err } - specs = append(specs, dspec...) - l = applyDeviceChange(l, dspec) - d.Set("reboot_required", true) + + c.Spec = append(c.Spec, dspec...) + c.VirtualDevice = applyDeviceChange(c.VirtualDevice, dspec) + } + log.Printf("VirtualMachine: Shared PCI device specs created") + return nil +} + +// SharedPciApplyOperation checks for changes in a virtual machine's +// Shared PCI device and creates config specs to apply to the +// virtual machine. +func SharedPciApplyOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) { + log.Printf("[DEBUG] SharedPciApplyOperation: Looking for shared PCI device changes") + + // Get current (old) and new devices + old, newValue := d.GetChange("shared_pci_device_id") + oldDevIds := old.(*schema.Set) + newDevIds := newValue.(*schema.Set) + + // Compare + delDevs := oldDevIds.Difference(newDevIds) + addDevs := newDevIds.Difference(oldDevIds) + + // Create base apply config + applyConfig := &pciApplyConfig{ + Client: c, + ResourceData: d, + Spec: []types.BaseVirtualDeviceConfigSpec{}, + VirtualDevice: l, + } + + // If there are no changes, return as is + if addDevs.Len() == 0 && delDevs.Len() == 0 { + log.Printf("[DEBUG] SharedPciApplyOperation: No shared PCI device additions/deletions") + return applyConfig.VirtualDevice, applyConfig.Spec, nil + } + + // Set reboot + _ = d.Set("reboot_required", true) + + // Add new Shared PCI devices + log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device additions", + addDevs.Len()) + err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd) + if err != nil { + return nil, nil, err } - return l, specs, nil + // Remove deleted Shared PCI devices + log.Printf("[DEBUG] SharedPciApplyOperation: Identified %d shared PCI device deletions", + delDevs.Len()) + err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove) + if err != nil { + return nil, nil, err + } + + return applyConfig.VirtualDevice, applyConfig.Spec, nil +} + +// SharedPciPostCloneOperation normalizes the Shared PCI devices +// on a newly-cloned virtual machine and outputs any necessary device change +// operations. It also sets the state in advance of the post-create read. +func SharedPciPostCloneOperation(d *schema.ResourceData, c *govmomi.Client, l object.VirtualDeviceList) (object.VirtualDeviceList, []types.BaseVirtualDeviceConfigSpec, error) { + log.Printf("[DEBUG] SharedPCIPostCloneOperation: Looking for shared PCI device changes post-clone") + + // Get current (old) and new devices + old, newValue := d.GetChange("shared_pci_device_id") + oldDevIds := old.(*schema.Set) + newDevIds := newValue.(*schema.Set) + + // Compare + delDevs := oldDevIds.Difference(newDevIds) + addDevs := newDevIds.Difference(oldDevIds) + + // Create base apply config + applyConfig := &pciApplyConfig{ + Client: c, + ResourceData: d, + Spec: []types.BaseVirtualDeviceConfigSpec{}, + VirtualDevice: l, + } + + // If there are no changes, return as is + if addDevs.Len() <= 0 && delDevs.Len() <= 0 { + log.Printf("[DEBUG] SharedPCIPostCloneOperation: No shared PCI device additions/deletions post-clone") + return applyConfig.VirtualDevice, applyConfig.Spec, nil + } + + // Add new Shared PCI devices + log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device additions post-clone", + addDevs.Len()) + err := applyConfig.modifyVirtualSharedPciDevices(addDevs, types.VirtualDeviceConfigSpecOperationAdd) + if err != nil { + return nil, nil, err + } + + // Remove deleted Shared PCI devices + log.Printf("[DEBUG] SharedPCIPostCloneOperation: Identified %d shared PCI device deletions post-clone", + delDevs.Len()) + err = applyConfig.modifyVirtualSharedPciDevices(delDevs, types.VirtualDeviceConfigSpecOperationRemove) + if err != nil { + return nil, nil, err + } + return applyConfig.VirtualDevice, applyConfig.Spec, nil } diff --git a/vsphere/resource_vsphere_file.go b/vsphere/resource_vsphere_file.go index 57bc4dade..1a128cc51 100644 --- a/vsphere/resource_vsphere_file.go +++ b/vsphere/resource_vsphere_file.go @@ -149,13 +149,10 @@ func createDirectory(datastoreFileManager *object.DatastoreFileManager, f *file) // fileUpload - upload file to a vSphere datastore func fileUpload(client *govmomi.Client, dc *object.Datacenter, ds *object.Datastore, source, destination string) error { - dsurl, err := ds.URL(context.TODO(), dc, destination) - if err != nil { - return err - } + dsurl := ds.NewURL(destination) p := soap.DefaultUpload - err = client.Client.UploadFile(context.TODO(), source, dsurl, &p) + err := client.Client.UploadFile(context.TODO(), source, dsurl, &p) if err != nil { return err } diff --git a/vsphere/resource_vsphere_virtual_machine.go b/vsphere/resource_vsphere_virtual_machine.go index d6b44dcfa..fea0d4501 100644 --- a/vsphere/resource_vsphere_virtual_machine.go +++ b/vsphere/resource_vsphere_virtual_machine.go @@ -244,9 +244,9 @@ func resourceVSphereVirtualMachine() *schema.Resource { Elem: &schema.Schema{Type: schema.TypeString}, }, "shared_pci_device_id": { - Type: schema.TypeString, + Type: schema.TypeSet, Optional: true, - Description: "Id of Shared PCI passthrough device, 'grid_rtx8000-8q'", + Description: "A list of Shared PCI passthrough device, 'grid_rtx8000-8q'", Elem: &schema.Schema{Type: schema.TypeString}, }, "clone": { @@ -515,21 +515,23 @@ func resourceVSphereVirtualMachineRead(d *schema.ResourceData, meta interface{}) // Read the virtual machine PCI passthrough devices var pciDevs []string + var sharedPciDevs []string for _, dev := range vprops.Config.Hardware.Device { if pci, ok := dev.(*types.VirtualPCIPassthrough); ok { - if pciBacking, ok := pci.Backing.(*types.VirtualPCIPassthroughDeviceBackingInfo); ok { - devId := pciBacking.Id + switch t := pci.Backing.(type) { + case *types.VirtualPCIPassthroughDeviceBackingInfo: + devId := t.Id pciDevs = append(pciDevs, devId) - } else { - if pciBacking, ok := pci.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo); ok { - err = d.Set("shared_pci_device_id", pciBacking.Vgpu) - if err != nil { - return err - } - } else { - log.Printf("[WARN] Ignoring VM %q VirtualPCIPassthrough device with backing type of %T", - vm.InventoryPath, pci.Backing) - } + log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T", + vm.InventoryPath, devId, pci.Backing) + case *types.VirtualPCIPassthroughVmiopBackingInfo: + dev := t.Vgpu + sharedPciDevs = append(sharedPciDevs, dev) + log.Printf("[DEBUG] Identified VM %q VirtualPCIPassthrough device %s with backing type of %T", + vm.InventoryPath, dev, pci.Backing) + default: + log.Printf("[WARN] Ignoring VM %q VirtualPCIPassthrough device with backing type of %T", + vm.InventoryPath, pci.Backing) } } } @@ -537,6 +539,10 @@ func resourceVSphereVirtualMachineRead(d *schema.ResourceData, meta interface{}) if err != nil { return err } + err = d.Set("shared_pci_device_id", sharedPciDevs) + if err != nil { + return err + } // Perform pending device read operations. devices := object.VirtualDeviceList(vprops.Config.Hardware.Device) @@ -1675,6 +1681,17 @@ func resourceVSphereVirtualMachinePostDeployChanges(d *schema.ResourceData, meta ) } cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...) + // Shared PCI devices + devices, delta, err = virtualdevice.SharedPciPostCloneOperation(d, client, devices) + if err != nil { + return resourceVSphereVirtualMachineRollbackCreate( + d, + meta, + vm, + fmt.Errorf("error processing shared PCI device changes post-clone: %s", err), + ) + } + cfgSpec.DeviceChange = virtualdevice.AppendDeviceChangeSpec(cfgSpec.DeviceChange, delta...) log.Printf("[DEBUG] %s: Final device list: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceListString(devices)) log.Printf("[DEBUG] %s: Final device change cfgSpec: %s", resourceVSphereVirtualMachineIDString(d), virtualdevice.DeviceChangeString(cfgSpec.DeviceChange)) @@ -1978,8 +1995,8 @@ func applyVirtualDevices(d *schema.ResourceData, c *govmomi.Client, l object.Vir return nil, err } spec = virtualdevice.AppendDeviceChangeSpec(spec, delta...) - // Shared PCI passthrough device - l, delta, err = virtualdevice.SharedPciPassthroughApplyOperation(d, c, l) + // Shared PCI device + l, delta, err = virtualdevice.SharedPciApplyOperation(d, c, l) if err != nil { return nil, err } diff --git a/website/docs/r/virtual_machine.html.markdown b/website/docs/r/virtual_machine.html.markdown index 36fc9d950..84c9ceca3 100644 --- a/website/docs/r/virtual_machine.html.markdown +++ b/website/docs/r/virtual_machine.html.markdown @@ -616,6 +616,8 @@ The following options are general virtual machine and provider workflow options: * `clone` - (Optional) When specified, the virtual machine will be created as a clone of a specified template. Optional customization options can be submitted for the resource. See [creating a virtual machine from a template](#creating-a-virtual-machine-from-a-template) for more information. +~> **NOTE:** Cloning requires vCenter Server and is not supported on direct ESXi host connections. + * `extra_config_reboot_required` - (Optional) Allow the virtual machine to be rebooted when a change to `extra_config` occurs. Default: `true`. * `custom_attributes` - (Optional) Map of custom attribute ids to attribute value strings to set for virtual machine. Please refer to the [`vsphere_custom_attributes`][docs-setting-custom-attributes] resource for more information on setting custom attributes. @@ -661,14 +663,10 @@ The following options are general virtual machine and provider workflow options: * `network_interface` - (Required) A specification for a virtual NIC on the virtual machine. See [network interface options](#network-interface-options) for more information. -* `pci_device_id` - (Optional) List of host PCI device IDs in which to create PCI passthroughs. - -* `shared_pci_device_id` - (Optional) A shared PCI device ID to create PCI passthrough. - -~> **NOTE:** Cloning requires vCenter Server and is not supported on direct ESXi host connections. - * `ovf_deploy` - (Optional) When specified, the virtual machine will be deployed from the provided OVF/OVA template. See [creating a virtual machine from an OVF/OVA template](#creating-a-virtual-machine-from-an-ovf-ova-template) for more information. +* `pci_device_id` - (Optional) List of host PCI device IDs in which to create PCI passthroughs. + * `replace_trigger` - (Optional) Triggers replacement of resource whenever it changes. For example, `replace_trigger = sha256(format("%s-%s",data.template_file.cloud_init_metadata.rendered,data.template_file.cloud_init_userdata.rendered))` will fingerprint the changes in cloud-init metadata and userdata templates. This will enable a replacement of the resource whenever the dependant template renders a new configuration. (Forces a replacement.) @@ -685,6 +683,16 @@ For example, `replace_trigger = sha256(format("%s-%s",data.template_file.cloud_i * `scsi_bus_sharing` - (Optional) The type of SCSI bus sharing for the virtual machine SCSI controller. One of `physicalSharing`, `virtualSharing`, and `noSharing`. Default: `noSharing`. +* `shared_pci_device_id` - (Optional) List of shared PCI device(s) to create +shared PCI passthrough. + +For example, attaching a vGPU to the virtual machine: +`shared_pci_device_id = ["grid_a100d-40c"]` + +~> **NOTE:** The use of vGPU requires that the VM has memory reservation set equal +to the amount of provisioned memory. This can be accomplished by leveraging +the [`memory_reservation`](#memory_reservation) option. + * `storage_policy_id` - (Optional) The ID of the storage policy to assign to the home directory of a virtual machine. * `tags` - (Optional) The IDs of any tags to attach to this resource. Please refer to the [`vsphere_tag`][docs-applying-tags] resource for more information on applying tags to virtual machine resources.