diff --git a/.chloggen/feat_faster-vcenter-vm-network-calls.yaml b/.chloggen/feat_faster-vcenter-vm-network-calls.yaml new file mode 100644 index 000000000000..c5189c091be3 --- /dev/null +++ b/.chloggen/feat_faster-vcenter-vm-network-calls.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: vcenterreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Changes process for collecting VMs & VM perf metrics used by the `vccenterreceiver` to be more efficient (one call now for all VMs)" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [31837] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/receiver/vcenterreceiver/client.go b/receiver/vcenterreceiver/client.go index 17df6716d154..6d459e164961 100644 --- a/receiver/vcenterreceiver/client.go +++ b/receiver/vcenterreceiver/client.go @@ -13,7 +13,9 @@ import ( "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/performance" "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/view" "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" vt "github.com/vmware/govmomi/vim25/types" ) @@ -24,6 +26,7 @@ type vcenterClient struct { finder *find.Finder pc *property.Collector pm *performance.Manager + vm *view.Manager cfg *Config } @@ -69,6 +72,7 @@ func (vc *vcenterClient) EnsureConnection(ctx context.Context) error { vc.pc = property.DefaultCollector(vc.vimDriver) vc.finder = find.NewFinder(vc.vimDriver) vc.pm = performance.NewManager(vc.vimDriver) + vc.vm = view.NewManager(vc.vimDriver) return nil } @@ -89,6 +93,7 @@ func (vc *vcenterClient) Datacenters(ctx context.Context) ([]*object.Datacenter, return datacenters, nil } +// Computes returns the ComputeResources (and ClusterComputeResources) of the vSphere SDK for a given datacenter func (vc *vcenterClient) Computes(ctx context.Context, datacenter *object.Datacenter) ([]*object.ComputeResource, error) { vc.finder = vc.finder.SetDatacenter(datacenter) computes, err := vc.finder.ComputeResourceList(ctx, "*") @@ -98,7 +103,7 @@ func (vc *vcenterClient) Computes(ctx context.Context, datacenter *object.Datace return computes, nil } -// ResourcePools returns the resourcePools in the vSphere SDK +// ResourcePools returns the ResourcePools in the vSphere SDK func (vc *vcenterClient) ResourcePools(ctx context.Context) ([]*object.ResourcePool, error) { rps, err := vc.finder.ResourcePoolList(ctx, "*") if err != nil { @@ -107,12 +112,35 @@ func (vc *vcenterClient) ResourcePools(ctx context.Context) ([]*object.ResourceP return rps, err } -func (vc *vcenterClient) VMs(ctx context.Context) ([]*object.VirtualMachine, error) { - vms, err := vc.finder.VirtualMachineList(ctx, "*") +func (vc *vcenterClient) VMs(ctx context.Context) ([]mo.VirtualMachine, error) { + v, err := vc.vm.CreateContainerView(ctx, vc.vimDriver.ServiceContent.RootFolder, []string{"VirtualMachine"}, true) if err != nil { - return nil, fmt.Errorf("unable to retrieve vms: %w", err) + return nil, fmt.Errorf("unable to retrieve VMs: %w", err) } - return vms, err + + var vms []mo.VirtualMachine + err = v.Retrieve(ctx, []string{"VirtualMachine"}, []string{ + "config.hardware.numCPU", + "config.instanceUuid", + "runtime.powerState", + "runtime.maxCpuUsage", + "summary.quickStats.guestMemoryUsage", + "summary.quickStats.balloonedMemory", + "summary.quickStats.swappedMemory", + "summary.quickStats.ssdSwappedMemory", + "summary.quickStats.overallCpuUsage", + "summary.config.memorySizeMB", + "summary.config.name", + "summary.storage.committed", + "summary.storage.uncommitted", + "summary.runtime.host", + "resourcePool", + }, &vms) + if err != nil { + return nil, fmt.Errorf("unable to retrieve VMs: %w", err) + } + + return vms, nil } type perfSampleResult struct { @@ -120,6 +148,11 @@ type perfSampleResult struct { results []performance.EntityMetric } +type perfMetricsQueryResult struct { + counters map[string]*vt.PerfCounterInfo + resultsByMoRef map[string]*performance.EntityMetric +} + func (vc *vcenterClient) performanceQuery( ctx context.Context, spec vt.PerfQuerySpec, @@ -147,3 +180,36 @@ func (vc *vcenterClient) performanceQuery( results: result, }, nil } + +func (vc *vcenterClient) perfMetricsQuery( + ctx context.Context, + spec vt.PerfQuerySpec, + names []string, + objs []vt.ManagedObjectReference, +) (*perfMetricsQueryResult, error) { + if vc.pm == nil { + return &perfMetricsQueryResult{}, nil + } + vc.pm.Sort = true + sample, err := vc.pm.SampleByName(ctx, spec, names, objs) + if err != nil { + return nil, err + } + result, err := vc.pm.ToMetricSeries(ctx, sample) + if err != nil { + return nil, err + } + counterInfoByName, err := vc.pm.CounterInfoByName(ctx) + if err != nil { + return nil, err + } + + resultsByMoRef := map[string]*performance.EntityMetric{} + for i := range result { + resultsByMoRef[result[i].Entity.Value] = &result[i] + } + return &perfMetricsQueryResult{ + counters: counterInfoByName, + resultsByMoRef: resultsByMoRef, + }, nil +} diff --git a/receiver/vcenterreceiver/client_test.go b/receiver/vcenterreceiver/client_test.go index fb3cf28ce1a3..c9f2ae7837f7 100644 --- a/receiver/vcenterreceiver/client_test.go +++ b/receiver/vcenterreceiver/client_test.go @@ -13,6 +13,7 @@ import ( "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/simulator" + "github.com/vmware/govmomi/view" "github.com/vmware/govmomi/vim25" "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configtls" @@ -48,10 +49,12 @@ func TestGetResourcePools(t *testing.T) { func TestGetVMs(t *testing.T) { simulator.Test(func(ctx context.Context, c *vim25.Client) { + viewManager := view.NewManager(c) finder := find.NewFinder(c) client := vcenterClient{ vimDriver: c, finder: finder, + vm: viewManager, } vms, err := client.VMs(ctx) require.NoError(t, err) diff --git a/receiver/vcenterreceiver/internal/mockserver/client_mock.go b/receiver/vcenterreceiver/internal/mockserver/client_mock.go index 4cddd99be2ab..5c11703b2079 100644 --- a/receiver/vcenterreceiver/internal/mockserver/client_mock.go +++ b/receiver/vcenterreceiver/internal/mockserver/client_mock.go @@ -79,6 +79,8 @@ func routeBody(t *testing.T, requestType string, body map[string]any) ([]byte, e return routeRetreiveProperties(t, body) case "QueryPerf": return routePerformanceQuery(t, body) + case "CreateContainerView": + return loadResponse("create-container-view.xml") } return []byte{}, errNotFound @@ -163,7 +165,7 @@ func routeRetreiveProperties(t *testing.T, body map[string]any) ([]byte, error) case contentType == "HostSystem": if ps, ok := propSet["pathSet"].([]any); ok { for _, v := range ps { - if v == "summary.hardware" { + if v == "summary.hardware" || v == "summary.hardware.cpuMhz" { return loadResponse("host-properties.xml") } } @@ -179,26 +181,8 @@ func routeRetreiveProperties(t *testing.T, body map[string]any) ([]byte, error) } - case content == "group-v4" && contentType == "Folder": - if propSetArray { - return loadResponse("vm-group.xml") - } - if propSet == nil { - return loadResponse("vm-folder.xml") - } - return loadResponse("vm-folder-parent.xml") - - case content == "vm-1040" && contentType == "VirtualMachine": - if propSet["pathSet"] == "summary.runtime.host" { - return loadResponse("vm-host.xml") - } - if propSet["pathSet"] == "resourcePool" { - return loadResponse("vm-resource-pool.xml") - } - return loadResponse("vm-properties.xml") - - case (content == "group-v1034" || content == "group-v1001") && contentType == "Folder": - return loadResponse("vm-empty-folder.xml") + case contentType == "ContainerView" && propSet["type"] == "VirtualMachine": + return loadResponse("vm-default-properties.xml") case contentType == "ResourcePool": if ps, ok := propSet["pathSet"].([]any); ok { @@ -238,7 +222,11 @@ func routeRetreiveProperties(t *testing.T, body map[string]any) ([]byte, error) func routePerformanceQuery(t *testing.T, body map[string]any) ([]byte, error) { queryPerf := body["QueryPerf"].(map[string]any) require.NotNil(t, queryPerf) - querySpec := queryPerf["querySpec"].(map[string]any) + querySpec, ok := queryPerf["querySpec"].(map[string]any) + if !ok { + querySpecs := queryPerf["querySpec"].([]any) + querySpec = querySpecs[0].(map[string]any) + } entity := querySpec["entity"].(map[string]any) switch entity["-type"] { case "HostSystem": diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-empty-folder.xml b/receiver/vcenterreceiver/internal/mockserver/responses/create-container-view.xml similarity index 55% rename from receiver/vcenterreceiver/internal/mockserver/responses/vm-empty-folder.xml rename to receiver/vcenterreceiver/internal/mockserver/responses/create-container-view.xml index 26226f9bb581..980d9396f76a 100644 --- a/receiver/vcenterreceiver/internal/mockserver/responses/vm-empty-folder.xml +++ b/receiver/vcenterreceiver/internal/mockserver/responses/create-container-view.xml @@ -1,6 +1,8 @@ - + + session[5279c748-1a9c-e5f0-cc05-71b15da3a8a6]529d3b1c-0a03-30d0-f0c8-8cae2cc07d25 + - + \ No newline at end of file diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/host-properties.xml b/receiver/vcenterreceiver/internal/mockserver/responses/host-properties.xml index 1d29fbd37010..f44607e2b161 100644 --- a/receiver/vcenterreceiver/internal/mockserver/responses/host-properties.xml +++ b/receiver/vcenterreceiver/internal/mockserver/responses/host-properties.xml @@ -4,6 +4,14 @@ host-1002 + + name + esxi-27971.cf5e88ac.australia-southeast1.gve.goog + + + summary.hardware.cpuMhz + 2593 + config diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-default-properties.xml b/receiver/vcenterreceiver/internal/mockserver/responses/vm-default-properties.xml new file mode 100644 index 000000000000..19fe14aa1e48 --- /dev/null +++ b/receiver/vcenterreceiver/internal/mockserver/responses/vm-default-properties.xml @@ -0,0 +1,70 @@ + + + + + + vm-1040 + + config.hardware.numCPU + 4 + + + config.instanceUuid + 5000bbe0-993e-5813-c56a-198eaa62fb61 + + + resourcePool + resgroup-9 + + + runtime.maxCpuUsage + 10372 + + + runtime.powerState + poweredOn + + + summary.config.memorySizeMB + 16384 + + + summary.config.name + CentOS 7 + + + summary.quickStats.balloonedMemory + 0 + + + summary.quickStats.guestMemoryUsage + 163 + + + summary.quickStats.overallCpuUsage + 12 + + + summary.quickStats.ssdSwappedMemory + 0 + + + summary.quickStats.swappedMemory + 0 + + + summary.runtime.host + host-1002 + + + summary.storage.committed + 16311648256 + + + summary.storage.uncommitted + 258847277056 + + + + + diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-folder-parent.xml b/receiver/vcenterreceiver/internal/mockserver/responses/vm-folder-parent.xml deleted file mode 100644 index 1d683e80ef9f..000000000000 --- a/receiver/vcenterreceiver/internal/mockserver/responses/vm-folder-parent.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - group-v4 - - name - vm - - - parent - datacenter-3 - - - - datacenter-3 - - name - Datacenter - - - parent - group-d1 - - - - group-d1 - - name - Datacenters - - - - - diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-folder.xml b/receiver/vcenterreceiver/internal/mockserver/responses/vm-folder.xml deleted file mode 100644 index e52662442746..000000000000 --- a/receiver/vcenterreceiver/internal/mockserver/responses/vm-folder.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - vm-6005 - - name - loadbalancer - - - - vm-6004 - - name - debian - - - - group-v1034 - - childType - - Folder - VirtualMachine - VirtualApp - - - - name - HCX Management VMs - - - - group-v1001 - - childType - - Folder - VirtualMachine - VirtualApp - - - - name - Discovered virtual machine - - - - group-v1022 - - childType - - Folder - VirtualMachine - VirtualApp - - - - name - Workload VMs - - - - - diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-group.xml b/receiver/vcenterreceiver/internal/mockserver/responses/vm-group.xml deleted file mode 100644 index 270c3b85867f..000000000000 --- a/receiver/vcenterreceiver/internal/mockserver/responses/vm-group.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - vm-1040 - - name - CentOS 7 - - - - - diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-host.xml b/receiver/vcenterreceiver/internal/mockserver/responses/vm-host.xml deleted file mode 100644 index 0880d8ed974d..000000000000 --- a/receiver/vcenterreceiver/internal/mockserver/responses/vm-host.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - vm-6005 - - summary.runtime.host - host-1002 - - - - - diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-properties.xml b/receiver/vcenterreceiver/internal/mockserver/responses/vm-properties.xml deleted file mode 100644 index 9f74918e57eb..000000000000 --- a/receiver/vcenterreceiver/internal/mockserver/responses/vm-properties.xml +++ /dev/null @@ -1,1304 +0,0 @@ - - - - - - vm-6004 - - resourcePool - resgroup-9 - - - config - - 2022-04-13T18:56:00.417958Z - 1970-01-01T00:00:00Z - debian - Debian GNU/Linux 11 (64-bit) - vmx-19 - 42001f4c-7517-8f34-7a25-82539bff5246 - 2022-04-13T17:59:31.644999Z - 5000bbe0-993e-5813-c56a-198eaa62fb61 - true - 564de59f-2b94-0b67-2754-ed2d4dc0a9e3 - - debian11_64Guest - - - [vsanDatastore] 830f5762-12e2-515b-102b-0c42a15b3c50/debian.vmx - [vsanDatastore] 830f5762-12e2-515b-102b-0c42a15b3c50/ - [vsanDatastore] 830f5762-12e2-515b-102b-0c42a15b3c50/ - [vsanDatastore] 830f5762-12e2-515b-102b-0c42a15b3c50/ - - - 0 - true - true - true - true - manual - true - false - - 0 - - - - false - true - false - false - release - any - false - false - false - automatic - hvAuto - powerOff - false - unset - false - false - - - soft - hard - soft - soft - hard - soft - checkpoint - - - 4 - 1 - 16384 - false - false - - 3000 - - - ISO [vsanDatastore] d7cda260-92d1-4894-9cf8-0c42a15b3c50/debian-11.3.0-amd64-DVD-1.iso - - - [vsanDatastore] d7cda260-92d1-4894-9cf8-0c42a15b3c50/debian-11.3.0-amd64-DVD-1.iso - datastore-1003 - - - true - true - false - ok - - 200 - 0 - - - 12000 - - - Device on the virtual machine PCI bus that provides support for the virtual machine communication interface - - - 32 - - 100 - 17 - -1677766074 - false - true - - - 200 - - - IDE 0 - - 0 - 3000 - - - 201 - - - IDE 1 - - 1 - - - 300 - - - PS2 controller 0 - - 0 - 600 - 700 - - - 100 - - - PCI controller 0 - - 0 - 500 - 12000 - 1000 - 4000 - - - 400 - - - SIO controller 0 - - 0 - - - 600 - - - Keyboard - - 300 - 0 - - - 700 - - - Pointing device; Device - - - - false - autodetect - - 300 - 1 - - - 500 - - - Video card - - 100 - 0 - 8192 - 1 - false - false - automatic - 262144 - - - 1000 - - - VMware paravirtual SCSI - - 100 - 3 - 0 - 2000 - true - noSharing - 7 - - - 2000 - - - 125,829,120 KB - - - [vsanDatastore] 830f5762-12e2-515b-102b-0c42a15b3c50/debian.vmdk - datastore-1003 - 840f5762-ac4a-1296-feca-0c42a15b3c50 - persistent - false - false - true - false - 6000C29c-e66c-8729-d0e0-4ded1be42148 - d400fbcadf15c3a3c8a07b6ffffffffe - false - sharingNone - - 1000 - 0 - 125829120 - 128849018880 - - 1000 - normal - - - -1 - - 1000 - normal - - 0 - - 9-2000 - false - - - 4000 - - - DVSwitch: 50 00 df 4b 09 9b 8b f7-d8 2e ae 6e 9f fb 35 0a - - - - 50 00 df 4b 09 9b 8b f7-d8 2e ae 6e 9f fb 35 0a - dvportgroup-4007 - 107b844e-b6d8-41a7-b239-45b3c1e40acb - 1649876160 - - - - unset - true - true - true - ok - - 100 - 7 - assigned - 00:50:56:80:8d:62 - true - - 0 - - 50 - normal - - -1 - - 25a979c8-e087-4c56-a1ee-c77f7ba8c5d1 - true - - - - 0 - false - -1 - - 4000 - normal - - - - 0 - false - -1 - - 163840 - normal - - 151 - - false - false - false - 16384 - 0 - - nvram - debian.nvram - - - svga.present - TRUE - - - pciBridge0.present - TRUE - - - pciBridge4.present - TRUE - - - pciBridge4.virtualDev - pcieRootPort - - - pciBridge4.functions - 8 - - - pciBridge5.present - TRUE - - - pciBridge5.virtualDev - pcieRootPort - - - pciBridge5.functions - 8 - - - pciBridge6.present - TRUE - - - pciBridge6.virtualDev - pcieRootPort - - - pciBridge6.functions - 8 - - - pciBridge7.present - TRUE - - - pciBridge7.virtualDev - pcieRootPort - - - pciBridge7.functions - 8 - - - hpet0.present - TRUE - - - sched.cpu.latencySensitivity - normal - - - vmware.tools.internalversion - 0 - - - vmware.tools.requiredversion - 11334 - - - migrate.hostLogState - none - - - migrate.migrationId - 0 - - - migrate.hostLog - debian-65890666.hlog - - - viv.moid - c005f507-5050-4e15-bed8-10f2b41de30b:vm-6004:bxZ8c0yeNQvzT6ujHqKLtSp6nwsY5RIOA6n0BvNXlDU= - - - ethernet0.pciSlotNumber - 192 - - - monitor.phys_bits_used - 45 - - - numa.autosize.cookie - 40012 - - - numa.autosize.vcpu.maxPerVirtualNode - 4 - - - pciBridge0.pciSlotNumber - 17 - - - pciBridge4.pciSlotNumber - 21 - - - pciBridge5.pciSlotNumber - 22 - - - pciBridge6.pciSlotNumber - 23 - - - pciBridge7.pciSlotNumber - 24 - - - sched.swap.derivedName - /vmfs/volumes/vsan:52a9fa9bb23554ec-600746f1f7361622/830f5762-12e2-515b-102b-0c42a15b3c50/debian-049d5a68.vswp - - - scsi0.pciSlotNumber - 160 - - - scsi0.sasWWID - 50 05 05 6c 75 17 8f 30 - - - scsi0:0.redo - - - - softPowerOff - FALSE - - - vmci0.pciSlotNumber - 32 - - - vmotion.checkpointFBSize - 8388608 - - - vmotion.checkpointSVGAPrimarySize - 8388608 - - - vmotion.svga.graphicsMemoryKB - 8192 - - - vmotion.svga.mobMaxSize - 8388608 - - - svga.guestBackedPrimaryAware - TRUE - - - vsanDatastore - /vmfs/volumes/vsan:52a9fa9bb23554ec-600746f1f7361622 - - inherit - - 0 - false - false - false - 10000 - - false - bios - 40 - false - false - false - false - - never - none - - false - 830f5762-12e2-515b-102b-0c42a15b3c50 - 890f5762-021b-22d5-a34b-0c42a15b3c50 - - false - - opportunistic - - 0 - unlocked - - - ftEncryptionOpportunistic - - false - - - - runtime - - - - false - vmNptIncompatibleHost - green - - 4000 - - host-1002 - connected - poweredOn - notConfigured - - true - - false - 2022-04-13T18:54:10Z - 0 - 10372 - 16384 - 0 - inactive - false - intel-cascadelake - false - - cpuid.lm - cpuid.lm - Num:Min:1 - - - cpuid.3dnprefetch - cpuid.3dnprefetch - Bool:Min:1 - - - cpuid.abm - cpuid.abm - Bool:Min:1 - - - cpuid.adx - cpuid.adx - Bool:Min:1 - - - cpuid.aes - cpuid.aes - Bool:Min:1 - - - cpuid.avx - cpuid.avx - Bool:Min:1 - - - cpuid.avx2 - cpuid.avx2 - Bool:Min:1 - - - cpuid.avx512bw - cpuid.avx512bw - Bool:Min:1 - - - cpuid.avx512cd - cpuid.avx512cd - Bool:Min:1 - - - cpuid.avx512dq - cpuid.avx512dq - Bool:Min:1 - - - cpuid.avx512f - cpuid.avx512f - Bool:Min:1 - - - cpuid.avx512vl - cpuid.avx512vl - Bool:Min:1 - - - cpuid.avx512vnni - cpuid.avx512vnni - Bool:Min:1 - - - cpuid.bmi1 - cpuid.bmi1 - Bool:Min:1 - - - cpuid.bmi2 - cpuid.bmi2 - Bool:Min:1 - - - cpuid.clflushopt - cpuid.clflushopt - Bool:Min:1 - - - cpuid.clwb - cpuid.clwb - Bool:Min:1 - - - cpuid.cmpxchg16b - cpuid.cmpxchg16b - Bool:Min:1 - - - cpuid.enfstrg - cpuid.enfstrg - Bool:Min:1 - - - cpuid.f16c - cpuid.f16c - Bool:Min:1 - - - cpuid.fcmd - cpuid.fcmd - Bool:Min:1 - - - cpuid.fma - cpuid.fma - Bool:Min:1 - - - cpuid.fsgsbase - cpuid.fsgsbase - Bool:Min:1 - - - cpuid.ibpb - cpuid.ibpb - Bool:Min:1 - - - cpuid.ibrs - cpuid.ibrs - Bool:Min:1 - - - cpuid.intel - cpuid.intel - Bool:Min:1 - - - cpuid.invpcid - cpuid.invpcid - Bool:Min:1 - - - cpuid.lahf64 - cpuid.lahf64 - Bool:Min:1 - - - cpuid.lm - cpuid.lm - Bool:Min:1 - - - cpuid.mdclear - cpuid.mdclear - Bool:Min:1 - - - cpuid.movbe - cpuid.movbe - Bool:Min:1 - - - cpuid.nx - cpuid.nx - Bool:Min:1 - - - cpuid.pcid - cpuid.pcid - Bool:Min:1 - - - cpuid.pclmulqdq - cpuid.pclmulqdq - Bool:Min:1 - - - cpuid.pdpe1gb - cpuid.pdpe1gb - Bool:Min:1 - - - cpuid.pku - cpuid.pku - Bool:Min:1 - - - cpuid.popcnt - cpuid.popcnt - Bool:Min:1 - - - cpuid.rdrand - cpuid.rdrand - Bool:Min:1 - - - cpuid.rdseed - cpuid.rdseed - Bool:Min:1 - - - cpuid.rdtscp - cpuid.rdtscp - Bool:Min:1 - - - cpuid.smap - cpuid.smap - Bool:Min:1 - - - cpuid.smep - cpuid.smep - Bool:Min:1 - - - cpuid.ss - cpuid.ss - Bool:Min:1 - - - cpuid.ssbd - cpuid.ssbd - Bool:Min:1 - - - cpuid.sse3 - cpuid.sse3 - Bool:Min:1 - - - cpuid.sse41 - cpuid.sse41 - Bool:Min:1 - - - cpuid.sse42 - cpuid.sse42 - Bool:Min:1 - - - cpuid.ssse3 - cpuid.ssse3 - Bool:Min:1 - - - cpuid.stibp - cpuid.stibp - Bool:Min:1 - - - cpuid.xcr0_master_hi16_zmm - cpuid.xcr0_master_hi16_zmm - Bool:Min:1 - - - cpuid.xcr0_master_opmask - cpuid.xcr0_master_opmask - Bool:Min:1 - - - cpuid.xcr0_master_pkru - cpuid.xcr0_master_pkru - Bool:Min:1 - - - cpuid.xcr0_master_sse - cpuid.xcr0_master_sse - Bool:Min:1 - - - cpuid.xcr0_master_ymm_h - cpuid.xcr0_master_ymm_h - Bool:Min:1 - - - cpuid.xcr0_master_zmm_h - cpuid.xcr0_master_zmm_h - Bool:Min:1 - - - cpuid.xgetbv_ecx1 - cpuid.xgetbv_ecx1 - Bool:Min:1 - - - cpuid.xsave - cpuid.xsave - Bool:Min:1 - - - cpuid.xsavec - cpuid.xsavec - Bool:Min:1 - - - cpuid.xsaveopt - cpuid.xsaveopt - Bool:Min:1 - - - cpuid.xsaves - cpuid.xsaves - Bool:Min:1 - - - misc.cpuidfaulting - misc.cpuidfaulting - Bool:Min:1 - - - misc.ibrs_all - misc.ibrs_all - Bool:Min:1 - - - misc.mds_no - misc.mds_no - Bool:Min:1 - - - misc.rdcl_no - misc.rdcl_no - Bool:Min:1 - - - misc.rsba_no - misc.rsba_no - Bool:Min:1 - - false - false - false - - - - summary - - vm-6004 - - - - false - vmNptIncompatibleHost - green - - 4000 - - host-1002 - connected - poweredOn - notConfigured - - true - - false - 2022-04-13T18:54:10Z - 0 - 10372 - 16384 - 0 - inactive - false - intel-cascadelake - false - - cpuid.lm - cpuid.lm - Num:Min:1 - - - cpuid.3dnprefetch - cpuid.3dnprefetch - Bool:Min:1 - - - cpuid.abm - cpuid.abm - Bool:Min:1 - - - cpuid.adx - cpuid.adx - Bool:Min:1 - - - cpuid.aes - cpuid.aes - Bool:Min:1 - - - cpuid.avx - cpuid.avx - Bool:Min:1 - - - cpuid.avx2 - cpuid.avx2 - Bool:Min:1 - - - cpuid.avx512bw - cpuid.avx512bw - Bool:Min:1 - - - cpuid.avx512cd - cpuid.avx512cd - Bool:Min:1 - - - cpuid.avx512dq - cpuid.avx512dq - Bool:Min:1 - - - cpuid.avx512f - cpuid.avx512f - Bool:Min:1 - - - cpuid.avx512vl - cpuid.avx512vl - Bool:Min:1 - - - cpuid.avx512vnni - cpuid.avx512vnni - Bool:Min:1 - - - cpuid.bmi1 - cpuid.bmi1 - Bool:Min:1 - - - cpuid.bmi2 - cpuid.bmi2 - Bool:Min:1 - - - cpuid.clflushopt - cpuid.clflushopt - Bool:Min:1 - - - cpuid.clwb - cpuid.clwb - Bool:Min:1 - - - cpuid.cmpxchg16b - cpuid.cmpxchg16b - Bool:Min:1 - - - cpuid.enfstrg - cpuid.enfstrg - Bool:Min:1 - - - cpuid.f16c - cpuid.f16c - Bool:Min:1 - - - cpuid.fcmd - cpuid.fcmd - Bool:Min:1 - - - cpuid.fma - cpuid.fma - Bool:Min:1 - - - cpuid.fsgsbase - cpuid.fsgsbase - Bool:Min:1 - - - cpuid.ibpb - cpuid.ibpb - Bool:Min:1 - - - cpuid.ibrs - cpuid.ibrs - Bool:Min:1 - - - cpuid.intel - cpuid.intel - Bool:Min:1 - - - cpuid.invpcid - cpuid.invpcid - Bool:Min:1 - - - cpuid.lahf64 - cpuid.lahf64 - Bool:Min:1 - - - cpuid.lm - cpuid.lm - Bool:Min:1 - - - cpuid.mdclear - cpuid.mdclear - Bool:Min:1 - - - cpuid.movbe - cpuid.movbe - Bool:Min:1 - - - cpuid.nx - cpuid.nx - Bool:Min:1 - - - cpuid.pcid - cpuid.pcid - Bool:Min:1 - - - cpuid.pclmulqdq - cpuid.pclmulqdq - Bool:Min:1 - - - cpuid.pdpe1gb - cpuid.pdpe1gb - Bool:Min:1 - - - cpuid.pku - cpuid.pku - Bool:Min:1 - - - cpuid.popcnt - cpuid.popcnt - Bool:Min:1 - - - cpuid.rdrand - cpuid.rdrand - Bool:Min:1 - - - cpuid.rdseed - cpuid.rdseed - Bool:Min:1 - - - cpuid.rdtscp - cpuid.rdtscp - Bool:Min:1 - - - cpuid.smap - cpuid.smap - Bool:Min:1 - - - cpuid.smep - cpuid.smep - Bool:Min:1 - - - cpuid.ss - cpuid.ss - Bool:Min:1 - - - cpuid.ssbd - cpuid.ssbd - Bool:Min:1 - - - cpuid.sse3 - cpuid.sse3 - Bool:Min:1 - - - cpuid.sse41 - cpuid.sse41 - Bool:Min:1 - - - cpuid.sse42 - cpuid.sse42 - Bool:Min:1 - - - cpuid.ssse3 - cpuid.ssse3 - Bool:Min:1 - - - cpuid.stibp - cpuid.stibp - Bool:Min:1 - - - cpuid.xcr0_master_hi16_zmm - cpuid.xcr0_master_hi16_zmm - Bool:Min:1 - - - cpuid.xcr0_master_opmask - cpuid.xcr0_master_opmask - Bool:Min:1 - - - cpuid.xcr0_master_pkru - cpuid.xcr0_master_pkru - Bool:Min:1 - - - cpuid.xcr0_master_sse - cpuid.xcr0_master_sse - Bool:Min:1 - - - cpuid.xcr0_master_ymm_h - cpuid.xcr0_master_ymm_h - Bool:Min:1 - - - cpuid.xcr0_master_zmm_h - cpuid.xcr0_master_zmm_h - Bool:Min:1 - - - cpuid.xgetbv_ecx1 - cpuid.xgetbv_ecx1 - Bool:Min:1 - - - cpuid.xsave - cpuid.xsave - Bool:Min:1 - - - cpuid.xsavec - cpuid.xsavec - Bool:Min:1 - - - cpuid.xsaveopt - cpuid.xsaveopt - Bool:Min:1 - - - cpuid.xsaves - cpuid.xsaves - Bool:Min:1 - - - misc.cpuidfaulting - misc.cpuidfaulting - Bool:Min:1 - - - misc.ibrs_all - misc.ibrs_all - Bool:Min:1 - - - misc.mds_no - misc.mds_no - Bool:Min:1 - - - misc.rdcl_no - misc.rdcl_no - Bool:Min:1 - - - misc.rsba_no - misc.rsba_no - Bool:Min:1 - - false - false - false - - - toolsNotInstalled - guestToolsNotInstalled - guestToolsNotInstalled - guestToolsNotRunning - - - debian - - [vsanDatastore] 830f5762-12e2-515b-102b-0c42a15b3c50/debian.vmx - 16384 - 0 - 0 - 4 - 1 - 1 - 42001f4c-7517-8f34-7a25-82539bff5246 - 5000bbe0-993e-5813-c56a-198eaa62fb61 - debian11_64Guest - Debian GNU/Linux 11 (64-bit) - - false - false - 0 - vmx-19 - - - 16311648256 - 258847277056 - 15992881152 - 2022-05-17T14:28:26.159606Z - - - 12 - 0 - 0 - 163 - 6599 - gray - 0 - 1904 - 10372 - 16535 - 6524 - 6524 - 0 - 0 - 0 - 76 - -1 - -1 - gray - 0 - 2925324 - 0 - - green - - - - - - diff --git a/receiver/vcenterreceiver/internal/mockserver/responses/vm-resource-pool.xml b/receiver/vcenterreceiver/internal/mockserver/responses/vm-resource-pool.xml deleted file mode 100644 index 95a3a4edd29e..000000000000 --- a/receiver/vcenterreceiver/internal/mockserver/responses/vm-resource-pool.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - vm-1040 - - resourcePool - host-1002 - - - - - diff --git a/receiver/vcenterreceiver/metrics.go b/receiver/vcenterreceiver/metrics.go index 1cca7ca7ea23..d61030e99b83 100644 --- a/receiver/vcenterreceiver/metrics.go +++ b/receiver/vcenterreceiver/metrics.go @@ -173,58 +173,34 @@ var vmPerfMetricList = []string{ "virtualDisk.totalReadLatency.average", } -func (v *vcenterMetricScraper) recordVMPerformance( - ctx context.Context, - vm mo.VirtualMachine, - errs *scrapererror.ScrapeErrors, -) { - spec := types.PerfQuerySpec{ - Entity: vm.Reference(), - Format: string(types.PerfFormatNormal), - // Just grabbing real time performance metrics of the current - // supported metrics by this receiver. If more are added we may need - // a system of making this user customizable or adapt to use a 5 minute interval per metric - IntervalId: int32(20), - } - - info, err := v.client.performanceQuery(ctx, spec, vmPerfMetricList, []types.ManagedObjectReference{vm.Reference()}) - if err != nil { - errs.AddPartial(1, err) - return - } - v.processVMPerformanceMetrics(info) -} - -func (v *vcenterMetricScraper) processVMPerformanceMetrics(info *perfSampleResult) { - for _, m := range info.results { - for _, val := range m.Value { - for j, nestedValue := range val.Value { - si := m.SampleInfo[j] - switch val.Name { - // Performance monitoring level 1 metrics - case "net.bytesTx.average": - v.mb.RecordVcenterVMNetworkThroughputDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionTransmitted, val.Instance) - case "net.bytesRx.average": - v.mb.RecordVcenterVMNetworkThroughputDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionReceived, val.Instance) - case "net.usage.average": - v.mb.RecordVcenterVMNetworkUsageDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, val.Instance) - case "net.packetsTx.summation": - v.mb.RecordVcenterVMNetworkPacketCountDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionTransmitted, val.Instance) - case "net.packetsRx.summation": - v.mb.RecordVcenterVMNetworkPacketCountDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionReceived, val.Instance) - - // Performance monitoring level 2 metrics required - case "disk.totalReadLatency.average": - v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionRead, metadata.AttributeDiskTypePhysical, val.Instance) - case "virtualDisk.totalReadLatency.average": - v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionRead, metadata.AttributeDiskTypeVirtual, val.Instance) - case "disk.totalWriteLatency.average": - v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionWrite, metadata.AttributeDiskTypePhysical, val.Instance) - case "virtualDisk.totalWriteLatency.average": - v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionWrite, metadata.AttributeDiskTypeVirtual, val.Instance) - case "disk.maxTotalLatency.latest": - v.mb.RecordVcenterVMDiskLatencyMaxDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, val.Instance) - } +func (v *vcenterMetricScraper) recordVMPerformanceMetrics(entityMetric *performance.EntityMetric) { + for _, val := range entityMetric.Value { + for j, nestedValue := range val.Value { + si := entityMetric.SampleInfo[j] + switch val.Name { + // Performance monitoring level 1 metrics + case "net.bytesTx.average": + v.mb.RecordVcenterVMNetworkThroughputDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionTransmitted, val.Instance) + case "net.bytesRx.average": + v.mb.RecordVcenterVMNetworkThroughputDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionReceived, val.Instance) + case "net.usage.average": + v.mb.RecordVcenterVMNetworkUsageDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, val.Instance) + case "net.packetsTx.summation": + v.mb.RecordVcenterVMNetworkPacketCountDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionTransmitted, val.Instance) + case "net.packetsRx.summation": + v.mb.RecordVcenterVMNetworkPacketCountDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeThroughputDirectionReceived, val.Instance) + + // Performance monitoring level 2 metrics required + case "disk.totalReadLatency.average": + v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionRead, metadata.AttributeDiskTypePhysical, val.Instance) + case "virtualDisk.totalReadLatency.average": + v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionRead, metadata.AttributeDiskTypeVirtual, val.Instance) + case "disk.totalWriteLatency.average": + v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionWrite, metadata.AttributeDiskTypePhysical, val.Instance) + case "virtualDisk.totalWriteLatency.average": + v.mb.RecordVcenterVMDiskLatencyAvgDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, metadata.AttributeDiskDirectionWrite, metadata.AttributeDiskTypeVirtual, val.Instance) + case "disk.maxTotalLatency.latest": + v.mb.RecordVcenterVMDiskLatencyMaxDataPoint(pcommon.NewTimestampFromTime(si.Timestamp), nestedValue, val.Instance) } } } diff --git a/receiver/vcenterreceiver/resources.go b/receiver/vcenterreceiver/resources.go new file mode 100644 index 000000000000..2ca69cc21025 --- /dev/null +++ b/receiver/vcenterreceiver/resources.go @@ -0,0 +1,31 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package vcenterreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/vcenterreceiver" + +import ( + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/vcenterreceiver/internal/metadata" +) + +func (v *vcenterMetricScraper) createVMResourceBuilder( + vm mo.VirtualMachine, + hs mo.HostSystem, + compute *object.ComputeResource, + rp *object.ResourcePool, +) *metadata.ResourceBuilder { + rb := v.mb.NewResourceBuilder() + rb.SetVcenterVMName(vm.Summary.Config.Name) + rb.SetVcenterVMID(vm.Config.InstanceUuid) + if compute.Reference().Type == "ClusterComputeResource" { + rb.SetVcenterClusterName(compute.Name()) + } + rb.SetVcenterHostName(hs.Name) + if rp != nil && rp.Name() != "" { + rb.SetVcenterResourcePoolName(rp.Name()) + rb.SetVcenterResourcePoolInventoryPath(rp.InventoryPath) + } + return rb +} diff --git a/receiver/vcenterreceiver/scraper.go b/receiver/vcenterreceiver/scraper.go index 9bfd0d27187e..8a6b2933a1f7 100644 --- a/receiver/vcenterreceiver/scraper.go +++ b/receiver/vcenterreceiver/scraper.go @@ -9,7 +9,9 @@ import ( "time" "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/performance" "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/featuregate" "go.opentelemetry.io/collector/pdata/pcommon" @@ -82,7 +84,6 @@ func (v *vcenterMetricScraper) scrape(ctx context.Context) (pmetric.Metrics, err if err := v.client.EnsureConnection(ctx); err != nil { return pmetric.NewMetrics(), fmt.Errorf("unable to connect to vSphere SDK: %w", err) } - err := v.collectDatacenters(ctx) // cleanup so any inventory moves are accounted for @@ -291,11 +292,27 @@ func (v *vcenterMetricScraper) collectVMs( compute *object.ComputeResource, errs *scrapererror.ScrapeErrors, ) (poweredOnVMs int64, poweredOffVMs int64) { + // Get all VMs with property data vms, err := v.client.VMs(ctx) if err != nil { errs.AddPartial(1, err) return } + + spec := types.PerfQuerySpec{ + Format: string(types.PerfFormatNormal), + // Just grabbing real time performance metrics of the current + // supported metrics by this receiver. If more are added we may need + // a system of making this user customizable or adapt to use a 5 minute interval per metric + IntervalId: int32(20), + } + vmMoRefs := getVMMoRefs(vms) + // Get all VM performance metrics + vmPerfMetrics, err := v.client.perfMetricsQuery(ctx, spec, vmPerfMetricList, vmMoRefs) + if err != nil { + errs.AddPartial(1, err) + } + for _, vm := range vms { computeName, ok := v.vmToComputeMap[vm.Reference().Value] if !ok { @@ -306,32 +323,18 @@ func (v *vcenterMetricScraper) collectVMs( continue } - var moVM mo.VirtualMachine - err := vm.Properties(ctx, vm.Reference(), []string{ - "config", - "runtime", - "summary", - }, &moVM) - - if err != nil { - errs.AddPartial(1, err) - continue - } - - if string(moVM.Runtime.PowerState) == "poweredOff" { + if string(vm.Runtime.PowerState) == "poweredOff" { poweredOffVMs++ } else { poweredOnVMs++ } - vmHost, err := vm.HostSystem(ctx) - if err != nil { - errs.AddPartial(1, err) - return - } - // vms are optional without a resource pool - rp, _ := vm.ResourcePool(ctx) + rpRef := vm.ResourcePool + var rp *object.ResourcePool + if rpRef != nil { + rp = object.NewResourcePool(v.client.vimDriver, *rpRef) + } if rp != nil { rpCompute, rpErr := rp.Owner(ctx) @@ -349,55 +352,58 @@ func (v *vcenterMetricScraper) collectVMs( } } - hostname, err := vmHost.ObjectName(ctx) - if err != nil { - errs.AddPartial(1, err) - return + if vm.Config == nil { + errs.AddPartial(1, fmt.Errorf("config empty for VM: %s", vm.Name)) + continue } + // Get related VM host info + hostRef := vm.Summary.Runtime.Host + if hostRef == nil { + errs.AddPartial(1, fmt.Errorf("no host for VM: %s", vm.Name)) + continue + } + vmHost := object.NewHostSystem(v.client.vimDriver, *hostRef) var hwSum mo.HostSystem err = vmHost.Properties(ctx, vmHost.Reference(), []string{ "name", - "summary.hardware", + "summary.hardware.cpuMhz", "vm", }, &hwSum) - if err != nil { errs.AddPartial(1, err) return } - if moVM.Config == nil { - errs.AddPartial(1, fmt.Errorf("vm config empty for %s", hostname)) - continue - } - vmUUID := moVM.Config.InstanceUuid - - v.collectVM(ctx, colTime, moVM, hwSum, errs) - rb := v.mb.NewResourceBuilder() - rb.SetVcenterVMName(vm.Name()) - rb.SetVcenterVMID(vmUUID) - if compute.Reference().Type == "ClusterComputeResource" { - rb.SetVcenterClusterName(compute.Name()) - } - rb.SetVcenterHostName(hostname) - if rp != nil && rp.Name() != "" { - rb.SetVcenterResourcePoolName(rp.Name()) - rb.SetVcenterResourcePoolInventoryPath(rp.InventoryPath) - } + perfMetrics := vmPerfMetrics.resultsByMoRef[vm.Reference().Value] + v.buildVMMetrics(colTime, vm, hwSum, perfMetrics) + rb := v.createVMResourceBuilder(vm, hwSum, compute, rp) v.mb.EmitForResource(metadata.WithResource(rb.Emit())) } + return poweredOnVMs, poweredOffVMs } -func (v *vcenterMetricScraper) collectVM( - ctx context.Context, +func (v *vcenterMetricScraper) buildVMMetrics( colTime pcommon.Timestamp, vm mo.VirtualMachine, hs mo.HostSystem, - errs *scrapererror.ScrapeErrors, + em *performance.EntityMetric, ) { v.recordVMUsages(colTime, vm, hs) - v.recordVMPerformance(ctx, vm, errs) + if em != nil { + v.recordVMPerformanceMetrics(em) + } +} + +func getVMMoRefs( + vms []mo.VirtualMachine, +) []types.ManagedObjectReference { + moRefs := []types.ManagedObjectReference{} + for _, vm := range vms { + moRefs = append(moRefs, vm.Reference()) + } + + return moRefs }